namespace :debshots do desc "Parse web server access logs and count visits of packages" task :accesslog2visits, [:logfile] => :environment do |t, args| Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::DEBUG REGEXP = /\[.+?\] Cache: (?\S+) (?\S+) (?\S+) (?\d+) (?\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?\/\S*) \"(?.+?)\" \"(?).+\"/ REGEXPS_URL = [ %r{/thumbnail/(?.+)}, %r{/thumbnail-404/(?.+)}, %r{/thumbnail-with-version/(?.+?)/}, %r{/screenshot/(?.+)}, %r{/screenshot-404/(?.+)}, %r{/screenshot-with-version/(?.+?)/}, %r{/package/(?.+)}, %r{/json/package/(?.+)}, ] packages={} logfile_name = args[:logfile] Rails.logger.info "Parsing log file #{logfile_name}" f = open(logfile_name, 'r') file_size = f.size f.each_line do |line| pkgname = nil if match = REGEXP.match(line) #p match['code'] url = match['url'].squeeze('/') REGEXPS_URL.each do |regexp| if url_match = regexp.match(url) pkgname = url_match['package'] break end end if pkgname packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1) end Rails.logger.debug "#{pkgname} / url=#{url}" else Rails.logger.error "No match: #{line}" end # TODO: URLs auswerten (thumbnail, …with-version, package…) -> paket-views-zähler sammeln # TODO: Cache-Hit-Ratio errechnen (nach größe der Response) end # Update packages with the count per package we summed up Package.transaction do packages.each do |pkg, count| Rails.logger.info "Package: #{pkg} Count: #{count}" db_pkg = Package.find_by_name pkg if db_pkg db_pkg.visits += count db_pkg.save! else Rails.logger.error "Package #{pkg} not found in database." end end end end end