namespace :debshots do
  desc "Parse web server access logs and count visits of packages"

  task :accesslog2visits, [:logfile] => :environment do |t, args|

    Rails.logger = Logger.new(STDOUT)
    Rails.logger.level = Logger::INFO
    # Rails.logger.level = Logger::DEBUG

    # Regular expression to match Nginx access log lines
    REGEXP = /\[.+?\] Cache: (?<cache>\S+) (?<backend>\S+) (?<resptime>\S+) (?<code>\d+) (?<size>\d+) \d+\.\d+\.\d+\.\d+(, \d+\.\d+\.\d+\.\d+)* (?<url>\/\S*) \"(?<referer>.*?)\" \"(?<agent>).*\"/

    # Regular subexpression to find the package name in different
    # kind of URLs.
    REGEXPS_URL = [
        %r{/thumbnail/(?<package>.+)},
        %r{/thumbnail-404/(?<package>.+)},
        %r{/thumbnail-with-version/(?<package>.+?)/},
        %r{/screenshot/(?<package>.+)},
        %r{/screenshot-404/(?<package>.+)},
        %r{/screenshot-with-version/(?<package>.+?)/},
        %r{/package/(?<package>.+)},
        %r{/json/package/(?<package>.+)},
    ]

    # Hash of packages. The key is the package name as found
    # in the URLs of the Nginx access log. The value is the
    # number of occurences that the package was found.
    packages={}

    # Get the logfile's path from the rake command line
    logfile_name = args[:logfile]
    Rails.logger.info "Parsing log file #{logfile_name}"
    f = open(logfile_name, 'r')

    # Get the total file size so that we know how far
    # we are through parsing the log file.
    file_size = f.size
    last_percent_done = 0

    f.each_line do |line|
      pkgname = nil

      # Show progress every 5%
      percent_done = (f.pos.to_f / f.size.to_f * 100).to_i
      if percent_done % 5 == 0
        if percent_done > last_percent_done
          last_percent_done = percent_done
          Rails.logger.info "Parsing... #{percent_done}%"
        end
      end

      if match = REGEXP.match(line)
          #p match['code']
          url = match['url'].squeeze('/')
          REGEXPS_URL.each do |regexp|
              if url_match = regexp.match(url)
                  pkgname = url_match['package']
                  break
              end
          end
          if pkgname
              packages[pkgname] = (packages[pkgname] ? packages[pkgname]+1 : 1)
          end
          Rails.logger.debug  "#{pkgname} / url=#{url}"
        else
          Rails.logger.error "Line did not match: #{line}"
        end
    end

    # Update packages with the count per package we summed up
    Rails.logger.info "Updating visits count in packages table"

    # It's faster to just go through all packages instead of updating them one by one

    last_percent_done = 0
    updated_packages = 0
    count_packages = Package.count
    Package.transaction do

      Package.find_each(batch_size: 1000) do |db_pkg|

        # Show progress every 5%
        percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i
        if percent_done % 5 == 0
          if percent_done > last_percent_done
            last_percent_done = percent_done
            Rails.logger.info "Updating... #{percent_done}%"
          end
        end
        updated_packages += 1

        if visits = packages[db_pkg.name]
          db_pkg.visits += visits
          db_pkg.save!
          Rails.logger.debug "Package: #{db_pkg.name}   Visits: #{visits}"
        else
          Rails.logger.debug "Package: #{db_pkg} (no visits counted)"
        end
      end # /find_in_batches

    end # /transaction

    Rails.logger.info "Done."
  end # /task
end # /namespace
