From b7e82757eb9410657eab907682c3c62a9eb948d1 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 11 Aug 2016 11:20:37 +0200 Subject: [PATCH] Speed improvement in updating database Instead of querying the database one package at a time I am now querying all packages and just update those who have a visits count. --- lib/tasks/accesslog2visits.rake | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/tasks/accesslog2visits.rake b/lib/tasks/accesslog2visits.rake index 6a4799e..2acf5f1 100644 --- a/lib/tasks/accesslog2visits.rake +++ b/lib/tasks/accesslog2visits.rake @@ -71,13 +71,17 @@ namespace :debshots do # Update packages with the count per package we summed up Rails.logger.info "Updating visits count in packages table" + # It's faster to just go through all packages instead of updating them one by one + last_percent_done = 0 updated_packages = 0 + count_packages = Package.count Package.transaction do - packages.each do |pkg, count| + + Package.find_each(batch_size: 1000) do |db_pkg| # Show progress every 5% - percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i + percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i if percent_done % 5 == 0 if percent_done > last_percent_done last_percent_done = percent_done @@ -86,17 +90,17 @@ namespace :debshots do end updated_packages += 1 - Rails.logger.debug "Package: #{pkg} Count: #{count}" - db_pkg = Package.find_by_name pkg - if db_pkg - db_pkg.visits += count + if visits = packages[db_pkg.name] + db_pkg.visits += visits db_pkg.save! + Rails.logger.debug "Package: #{db_pkg.name} Visits: #{visits}" else - Rails.logger.debug "Package #{pkg} not found in database." + Rails.logger.debug "Package: #{db_pkg} (no visits counted)" end - end - end + end # /find_in_batches + + end # /transaction Rails.logger.info "Done." - end -end + end # /task +end # /namespace