debshots/lib/tasks/accesslog2visits.rake
Christoph Haas b7e82757eb Speed improvement in updating database
Instead of querying the database one package at a time I am now querying
all packages and just update those who have a visits count.
2016-08-11 11:20:37 +02:00

106 lines
3.5 KiB
Ruby

namespace :debshots do
desc "Parse web server access logs and count visits of packages"
task :accesslog2visits, [:logfile] => :environment do |t, args|
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
# Rails.logger.level = Logger::DEBUG
# Regular expression to match Nginx access log lines
REGEXP = /\[.+?\] Cache: (?<cache>\S+) (?<backend>\S+) (?<resptime>\S+) (?<code>\d+) (?<size>\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?<url>\/\S*) \"(?<referer>.*?)\" \"(?<agent>).*\"/
# Regular subexpression to find the package name in different
# kind of URLs.
REGEXPS_URL = [
%r{/thumbnail/(?<package>.+)},
%r{/thumbnail-404/(?<package>.+)},
%r{/thumbnail-with-version/(?<package>.+?)/},
%r{/screenshot/(?<package>.+)},
%r{/screenshot-404/(?<package>.+)},
%r{/screenshot-with-version/(?<package>.+?)/},
%r{/package/(?<package>.+)},
%r{/json/package/(?<package>.+)},
]
# Hash of packages. The key is the package name as found
# in the URLs of the Nginx access log. The value is the
# number of occurences that the package was found.
packages={}
# Get the logfile's path from the rake command line
logfile_name = args[:logfile]
Rails.logger.info "Parsing log file #{logfile_name}"
f = open(logfile_name, 'r')
# Get the total file size so that we know how far
# we are through parsing the log file.
file_size = f.size
last_percent_done = 0
f.each_line do |line|
pkgname = nil
# Show progress every 5%
percent_done = (f.pos.to_f / f.size.to_f * 100).to_i
if percent_done % 5 == 0
if percent_done > last_percent_done
last_percent_done = percent_done
Rails.logger.info "Parsing... #{percent_done}%"
end
end
if match = REGEXP.match(line)
#p match['code']
url = match['url'].squeeze('/')
REGEXPS_URL.each do |regexp|
if url_match = regexp.match(url)
pkgname = url_match['package']
break
end
end
if pkgname
packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1)
end
Rails.logger.debug "#{pkgname} / url=#{url}"
else
Rails.logger.error "Line did not match: #{line}"
end
end
# Update packages with the count per package we summed up
Rails.logger.info "Updating visits count in packages table"
# It's faster to just go through all packages instead of updating them one by one
last_percent_done = 0
updated_packages = 0
count_packages = Package.count
Package.transaction do
Package.find_each(batch_size: 1000) do |db_pkg|
# Show progress every 5%
percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i
if percent_done % 5 == 0
if percent_done > last_percent_done
last_percent_done = percent_done
Rails.logger.info "Updating... #{percent_done}%"
end
end
updated_packages += 1
if visits = packages[db_pkg.name]
db_pkg.visits += visits
db_pkg.save!
Rails.logger.debug "Package: #{db_pkg.name} Visits: #{visits}"
else
Rails.logger.debug "Package: #{db_pkg} (no visits counted)"
end
end # /find_in_batches
end # /transaction
Rails.logger.info "Done."
end # /task
end # /namespace