102 lines
3.3 KiB
Ruby
102 lines
3.3 KiB
Ruby
namespace :debshots do
|
|
desc "Parse web server access logs and count visits of packages"
|
|
|
|
task :accesslog2visits, [:logfile] => :environment do |t, args|
|
|
|
|
Rails.logger = Logger.new(STDOUT)
|
|
Rails.logger.level = Logger::INFO
|
|
# Rails.logger.level = Logger::DEBUG
|
|
|
|
# Regular expression to match Nginx access log lines
|
|
REGEXP = /\[.+?\] Cache: (?<cache>\S+) (?<backend>\S+) (?<resptime>\S+) (?<code>\d+) (?<size>\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?<url>\/\S*) \"(?<referer>.*?)\" \"(?<agent>).*\"/
|
|
|
|
# Regular subexpression to find the package name in different
|
|
# kind of URLs.
|
|
REGEXPS_URL = [
|
|
%r{/thumbnail/(?<package>.+)},
|
|
%r{/thumbnail-404/(?<package>.+)},
|
|
%r{/thumbnail-with-version/(?<package>.+?)/},
|
|
%r{/screenshot/(?<package>.+)},
|
|
%r{/screenshot-404/(?<package>.+)},
|
|
%r{/screenshot-with-version/(?<package>.+?)/},
|
|
%r{/package/(?<package>.+)},
|
|
%r{/json/package/(?<package>.+)},
|
|
]
|
|
|
|
# Hash of packages. The key is the package name as found
|
|
# in the URLs of the Nginx access log. The value is the
|
|
# number of occurences that the package was found.
|
|
packages={}
|
|
|
|
# Get the logfile's path from the rake command line
|
|
logfile_name = args[:logfile]
|
|
Rails.logger.info "Parsing log file #{logfile_name}"
|
|
f = open(logfile_name, 'r')
|
|
|
|
# Get the total file size so that we know how far
|
|
# we are through parsing the log file.
|
|
file_size = f.size
|
|
last_percent_done = 0
|
|
|
|
f.each_line do |line|
|
|
pkgname = nil
|
|
|
|
# Show progress every 5%
|
|
percent_done = (f.pos.to_f / f.size.to_f * 100).to_i
|
|
if percent_done % 5 == 0
|
|
if percent_done > last_percent_done
|
|
last_percent_done = percent_done
|
|
Rails.logger.info "Parsing... #{percent_done}%"
|
|
end
|
|
end
|
|
|
|
if match = REGEXP.match(line)
|
|
#p match['code']
|
|
url = match['url'].squeeze('/')
|
|
REGEXPS_URL.each do |regexp|
|
|
if url_match = regexp.match(url)
|
|
pkgname = url_match['package']
|
|
break
|
|
end
|
|
end
|
|
if pkgname
|
|
packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1)
|
|
end
|
|
Rails.logger.debug "#{pkgname} / url=#{url}"
|
|
else
|
|
Rails.logger.error "Line did not match: #{line}"
|
|
end
|
|
end
|
|
|
|
# Update packages with the count per package we summed up
|
|
Rails.logger.info "Updating visits count in packages table"
|
|
|
|
last_percent_done = 0
|
|
updated_packages = 0
|
|
Package.transaction do
|
|
packages.each do |pkg, count|
|
|
|
|
# Show progress every 5%
|
|
percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i
|
|
if percent_done % 5 == 0
|
|
if percent_done > last_percent_done
|
|
last_percent_done = percent_done
|
|
Rails.logger.info "Updating... #{percent_done}%"
|
|
end
|
|
end
|
|
updated_packages += 1
|
|
|
|
Rails.logger.debug "Package: #{pkg} Count: #{count}"
|
|
db_pkg = Package.find_by_name pkg
|
|
if db_pkg
|
|
db_pkg.visits += count
|
|
db_pkg.save!
|
|
else
|
|
Rails.logger.debug "Package #{pkg} not found in database."
|
|
end
|
|
end
|
|
end
|
|
|
|
Rails.logger.info "Done."
|
|
end
|
|
end
|