Added rake task to count package visits from Nginx log files
Example call: rake debshots:accesslog2visits['cache-access-example.log']
This commit is contained in:
parent
5227c8d1fa
commit
04baeb73e7
1 changed files with 66 additions and 0 deletions
66
lib/tasks/accesslog2visits.rake
Normal file
66
lib/tasks/accesslog2visits.rake
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
namespace :debshots do
|
||||
desc "Parse web server access logs and count visits of packages"
|
||||
|
||||
task :accesslog2visits, [:logfile] => :environment do |t, args|
|
||||
|
||||
Rails.logger = Logger.new(STDOUT)
|
||||
Rails.logger.level = Logger::INFO
|
||||
# Rails.logger.level = Logger::DEBUG
|
||||
|
||||
REGEXP = /\[.+?\] Cache: (?<cache>\S+) (?<backend>\S+) (?<resptime>\S+) (?<code>\d+) (?<size>\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?<url>\/\S*) \"(?<referer>.+?)\" \"(?<agent>).+\"/
|
||||
REGEXPS_URL = [
|
||||
%r{/thumbnail/(?<package>.+)},
|
||||
%r{/thumbnail-404/(?<package>.+)},
|
||||
%r{/thumbnail-with-version/(?<package>.+?)/},
|
||||
%r{/screenshot/(?<package>.+)},
|
||||
%r{/screenshot-404/(?<package>.+)},
|
||||
%r{/screenshot-with-version/(?<package>.+?)/},
|
||||
%r{/package/(?<package>.+)},
|
||||
%r{/json/package/(?<package>.+)},
|
||||
]
|
||||
|
||||
packages={}
|
||||
|
||||
logfile_name = args[:logfile]
|
||||
Rails.logger.info "Parsing log file #{logfile_name}"
|
||||
f = open(logfile_name, 'r')
|
||||
file_size = f.size
|
||||
f.each_line do |line|
|
||||
pkgname = nil
|
||||
if match = REGEXP.match(line)
|
||||
#p match['code']
|
||||
url = match['url'].squeeze('/')
|
||||
REGEXPS_URL.each do |regexp|
|
||||
if url_match = regexp.match(url)
|
||||
pkgname = url_match['package']
|
||||
break
|
||||
end
|
||||
end
|
||||
if pkgname
|
||||
packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1)
|
||||
end
|
||||
Rails.logger.debug "#{pkgname} / url=#{url}"
|
||||
else
|
||||
Rails.logger.error "No match: #{line}"
|
||||
end
|
||||
|
||||
# TODO: URLs auswerten (thumbnail, …with-version, package…) -> paket-views-zähler sammeln
|
||||
# TODO: Cache-Hit-Ratio errechnen (nach größe der Response)
|
||||
end
|
||||
|
||||
# Update packages with the count per package we summed up
|
||||
Package.transaction do
|
||||
packages.each do |pkg, count|
|
||||
Rails.logger.info "Package: #{pkg} Count: #{count}"
|
||||
db_pkg = Package.find_by_name pkg
|
||||
if db_pkg
|
||||
db_pkg.visits += count
|
||||
db_pkg.save!
|
||||
else
|
||||
Rails.logger.error "Package #{pkg} not found in database."
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue