Progress counter in 5% steps added. Comments added.

lib/tasks/accesslog2visits.rake

@@ -10,6 +10,8 @@ namespace :debshots do
     # Regular expression to match Nginx access log lines
     REGEXP = /\[.+?\] Cache: (?<cache>\S+) (?<backend>\S+) (?<resptime>\S+) (?<code>\d+) (?<size>\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?<url>\/\S*) \"(?<referer>.*?)\" \"(?<agent>).*\"/
 
+    # Regular subexpression to find the package name in different
+    # kind of URLs.
     REGEXPS_URL = [
         %r{/thumbnail/(?<package>.+)},
         %r{/thumbnail-404/(?<package>.+)},
@@ -21,14 +23,33 @@ namespace :debshots do
         %r{/json/package/(?<package>.+)},
     ]
 
+    # Hash of packages. The key is the package name as found
+    # in the URLs of the Nginx access log. The value is the
+    # number of occurences that the package was found.
     packages={}
 
+    # Get the logfile's path from the rake command line
     logfile_name = args[:logfile]
     Rails.logger.info "Parsing log file #{logfile_name}"
     f = open(logfile_name, 'r')
+
+    # Get the total file size so that we know how far
+    # we are through parsing the log file.
     file_size = f.size
+    last_percent_done = 0
+
     f.each_line do |line|
       pkgname = nil
+
+      # Show progress every 5%
+      percent_done = (f.pos.to_f / f.size.to_f * 100).to_i
+      if percent_done % 5 == 0
+        if percent_done > last_percent_done
+          last_percent_done = percent_done
+          Rails.logger.info "Parsing... #{percent_done}%"
+        end
+      end
+
       if match = REGEXP.match(line)
           #p match['code']
           url = match['url'].squeeze('/')
@@ -43,26 +64,39 @@ namespace :debshots do
           end
           Rails.logger.debug  "#{pkgname} / url=#{url}"
         else
-          Rails.logger.error "No match: #{line}"
+          Rails.logger.error "Line did not match: #{line}"
         end
-
-        # TODO: URLs auswerten (thumbnail, …with-version, package…) -> paket-views-zähler sammeln
-        # TODO: Cache-Hit-Ratio errechnen (nach größe der Response)
     end
 
     # Update packages with the count per package we summed up
+    Rails.logger.info "Updating visits count in packages table"
+
+    last_percent_done = 0
+    updated_packages = 0
     Package.transaction do
       packages.each do |pkg, count|
-        Rails.logger.info "Package: #{pkg}   Count: #{count}"
+
+        # Show progress every 5%
+        percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i
+        if percent_done % 5 == 0
+          if percent_done > last_percent_done
+            last_percent_done = percent_done
+            Rails.logger.info "Updating... #{percent_done}%"
+          end
+        end
+        updated_packages += 1
+
+        Rails.logger.debug "Package: #{pkg}   Count: #{count}"
         db_pkg = Package.find_by_name pkg
         if db_pkg
           db_pkg.visits += count
           db_pkg.save!
         else
-          Rails.logger.error "Package #{pkg} not found in database."
+          Rails.logger.debug "Package #{pkg} not found in database."
         end
       end
     end
 
+    Rails.logger.info "Done."
   end
 end