chaas/debshots
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Speed improvement in updating database

Browse source 
Instead of querying the database one package at a time I am now querying
all packages and just update those who have a visits count.
This commit is contained in:
Christoph Haas 2016-08-11 11:20:37 +02:00
parent 2edae8fbc2
commit b7e82757eb
1 changed files with 15 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

26
lib/tasks/accesslog2visits.rake
View file

@ -71,13 +71,17 @@ namespace :debshots do
# Update packages with the count per package we summed up # Update packages with the count per package we summed up
Rails.logger.info "Updating visits count in packages table" Rails.logger.info "Updating visits count in packages table"
# It's faster to just go through all packages instead of updating them one by one
last_percent_done = 0 last_percent_done = 0
updated_packages = 0 updated_packages = 0
count_packages = Package.count
Package.transaction do Package.transaction do
packages.each do |pkg, count|
Package.find_each(batch_size: 1000) do |db_pkg|
# Show progress every 5% # Show progress every 5%
percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i
if percent_done % 5 == 0 if percent_done % 5 == 0
if percent_done > last_percent_done if percent_done > last_percent_done
last_percent_done = percent_done last_percent_done = percent_done
@ -86,17 +90,17 @@ namespace :debshots do
end end
updated_packages += 1 updated_packages += 1
Rails.logger.debug "Package: #{pkg} Count: #{count}" if visits = packages[db_pkg.name]
db_pkg = Package.find_by_name pkg db_pkg.visits += visits
if db_pkg
db_pkg.visits += count
db_pkg.save! db_pkg.save!
Rails.logger.debug "Package: #{db_pkg.name} Visits: #{visits}"
else else
Rails.logger.debug "Package #{pkg} not found in database." Rails.logger.debug "Package: #{db_pkg} (no visits counted)"
end end
end end # /find_in_batches
end
end # /transaction
Rails.logger.info "Done." Rails.logger.info "Done."
end end # /task
end end # /namespace