From 2edae8fbc24db8bcd8d42a954783ba6b7bc8bc70 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 11 Aug 2016 09:26:27 +0200 Subject: [PATCH] Progress counter in 5% steps added. Comments added. --- lib/tasks/accesslog2visits.rake | 46 ++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/lib/tasks/accesslog2visits.rake b/lib/tasks/accesslog2visits.rake index 042e0f4..6a4799e 100644 --- a/lib/tasks/accesslog2visits.rake +++ b/lib/tasks/accesslog2visits.rake @@ -10,6 +10,8 @@ namespace :debshots do # Regular expression to match Nginx access log lines REGEXP = /\[.+?\] Cache: (?\S+) (?\S+) (?\S+) (?\d+) (?\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?\/\S*) \"(?.*?)\" \"(?).*\"/ + # Regular subexpression to find the package name in different + # kind of URLs. REGEXPS_URL = [ %r{/thumbnail/(?.+)}, %r{/thumbnail-404/(?.+)}, @@ -21,14 +23,33 @@ namespace :debshots do %r{/json/package/(?.+)}, ] + # Hash of packages. The key is the package name as found + # in the URLs of the Nginx access log. The value is the + # number of occurences that the package was found. packages={} + # Get the logfile's path from the rake command line logfile_name = args[:logfile] Rails.logger.info "Parsing log file #{logfile_name}" f = open(logfile_name, 'r') + + # Get the total file size so that we know how far + # we are through parsing the log file. file_size = f.size + last_percent_done = 0 + f.each_line do |line| pkgname = nil + + # Show progress every 5% + percent_done = (f.pos.to_f / f.size.to_f * 100).to_i + if percent_done % 5 == 0 + if percent_done > last_percent_done + last_percent_done = percent_done + Rails.logger.info "Parsing... #{percent_done}%" + end + end + if match = REGEXP.match(line) #p match['code'] url = match['url'].squeeze('/') @@ -43,26 +64,39 @@ namespace :debshots do end Rails.logger.debug "#{pkgname} / url=#{url}" else - Rails.logger.error "No match: #{line}" + Rails.logger.error "Line did not match: #{line}" end - - # TODO: URLs auswerten (thumbnail, …with-version, package…) -> paket-views-zähler sammeln - # TODO: Cache-Hit-Ratio errechnen (nach größe der Response) end # Update packages with the count per package we summed up + Rails.logger.info "Updating visits count in packages table" + + last_percent_done = 0 + updated_packages = 0 Package.transaction do packages.each do |pkg, count| - Rails.logger.info "Package: #{pkg} Count: #{count}" + + # Show progress every 5% + percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i + if percent_done % 5 == 0 + if percent_done > last_percent_done + last_percent_done = percent_done + Rails.logger.info "Updating... #{percent_done}%" + end + end + updated_packages += 1 + + Rails.logger.debug "Package: #{pkg} Count: #{count}" db_pkg = Package.find_by_name pkg if db_pkg db_pkg.visits += count db_pkg.save! else - Rails.logger.error "Package #{pkg} not found in database." + Rails.logger.debug "Package #{pkg} not found in database." end end end + Rails.logger.info "Done." end end