namespace :debshots do desc "Parse web server access logs and count visits of packages" task :accesslog2visits, [:logfile] => :environment do |t, args| Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::DEBUG # Regular expression to match Nginx access log lines REGEXP = /\[.+?\] Cache: (?\S+) (?\S+) (?\S+) (?\d+) (?\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?\/\S*) \"(?.*?)\" \"(?).*\"/ # Regular subexpression to find the package name in different # kind of URLs. REGEXPS_URL = [ %r{/thumbnail/(?.+)}, %r{/thumbnail-404/(?.+)}, %r{/thumbnail-with-version/(?.+?)/}, %r{/screenshot/(?.+)}, %r{/screenshot-404/(?.+)}, %r{/screenshot-with-version/(?.+?)/}, %r{/package/(?.+)}, %r{/json/package/(?.+)}, ] # Hash of packages. The key is the package name as found # in the URLs of the Nginx access log. The value is the # number of occurences that the package was found. packages={} # Get the logfile's path from the rake command line logfile_name = args[:logfile] Rails.logger.info "Parsing log file #{logfile_name}" f = open(logfile_name, 'r') # Get the total file size so that we know how far # we are through parsing the log file. file_size = f.size last_percent_done = 0 f.each_line do |line| pkgname = nil # Show progress every 5% percent_done = (f.pos.to_f / f.size.to_f * 100).to_i if percent_done % 5 == 0 if percent_done > last_percent_done last_percent_done = percent_done Rails.logger.info "Parsing... #{percent_done}%" end end if match = REGEXP.match(line) #p match['code'] url = match['url'].squeeze('/') REGEXPS_URL.each do |regexp| if url_match = regexp.match(url) pkgname = url_match['package'] break end end if pkgname packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1) end Rails.logger.debug "#{pkgname} / url=#{url}" else Rails.logger.error "Line did not match: #{line}" end end # Update packages with the count per package we summed up Rails.logger.info "Updating visits count in packages table" # It's faster to just go through all packages instead of updating them one by one last_percent_done = 0 updated_packages = 0 count_packages = Package.count Package.transaction do Package.find_each(batch_size: 1000) do |db_pkg| # Show progress every 5% percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i if percent_done % 5 == 0 if percent_done > last_percent_done last_percent_done = percent_done Rails.logger.info "Updating... #{percent_done}%" end end updated_packages += 1 if visits = packages[db_pkg.name] db_pkg.visits += visits db_pkg.save! Rails.logger.debug "Package: #{db_pkg.name} Visits: #{visits}" else Rails.logger.debug "Package: #{db_pkg} (no visits counted)" end end # /find_in_batches end # /transaction Rails.logger.info "Done." end # /task end # /namespace