diff --git a/lib/tasks/accesslog2visits.rake b/lib/tasks/accesslog2visits.rake new file mode 100644 index 0000000..4eff02c --- /dev/null +++ b/lib/tasks/accesslog2visits.rake @@ -0,0 +1,66 @@ +namespace :debshots do + desc "Parse web server access logs and count visits of packages" + + task :accesslog2visits, [:logfile] => :environment do |t, args| + + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + REGEXP = /\[.+?\] Cache: (?\S+) (?\S+) (?\S+) (?\d+) (?\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?\/\S*) \"(?.+?)\" \"(?).+\"/ + REGEXPS_URL = [ + %r{/thumbnail/(?.+)}, + %r{/thumbnail-404/(?.+)}, + %r{/thumbnail-with-version/(?.+?)/}, + %r{/screenshot/(?.+)}, + %r{/screenshot-404/(?.+)}, + %r{/screenshot-with-version/(?.+?)/}, + %r{/package/(?.+)}, + %r{/json/package/(?.+)}, + ] + + packages={} + + logfile_name = args[:logfile] + Rails.logger.info "Parsing log file #{logfile_name}" + f = open(logfile_name, 'r') + file_size = f.size + f.each_line do |line| + pkgname = nil + if match = REGEXP.match(line) + #p match['code'] + url = match['url'].squeeze('/') + REGEXPS_URL.each do |regexp| + if url_match = regexp.match(url) + pkgname = url_match['package'] + break + end + end + if pkgname + packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1) + end + Rails.logger.debug "#{pkgname} / url=#{url}" + else + Rails.logger.error "No match: #{line}" + end + + # TODO: URLs auswerten (thumbnail, …with-version, package…) -> paket-views-zähler sammeln + # TODO: Cache-Hit-Ratio errechnen (nach größe der Response) + end + + # Update packages with the count per package we summed up + Package.transaction do + packages.each do |pkg, count| + Rails.logger.info "Package: #{pkg} Count: #{count}" + db_pkg = Package.find_by_name pkg + if db_pkg + db_pkg.visits += count + db_pkg.save! + else + Rails.logger.error "Package #{pkg} not found in database." + end + end + end + + end +end