Speed improvement in updating database

Instead of querying the database one package at a time I am now querying
all packages and just update those who have a visits count.
This commit is contained in:
Christoph Haas 2016-08-11 11:20:37 +02:00
parent 2edae8fbc2
commit b7e82757eb

View file

@ -71,13 +71,17 @@ namespace :debshots do
# Update packages with the count per package we summed up
Rails.logger.info "Updating visits count in packages table"
# It's faster to just go through all packages instead of updating them one by one
last_percent_done = 0
updated_packages = 0
count_packages = Package.count
Package.transaction do
packages.each do |pkg, count|
Package.find_each(batch_size: 1000) do |db_pkg|
# Show progress every 5%
percent_done = (updated_packages.to_f / packages.count.to_f * 100).to_i
percent_done = (updated_packages.to_f / count_packages.to_f * 100).to_i
if percent_done % 5 == 0
if percent_done > last_percent_done
last_percent_done = percent_done
@ -86,17 +90,17 @@ namespace :debshots do
end
updated_packages += 1
Rails.logger.debug "Package: #{pkg} Count: #{count}"
db_pkg = Package.find_by_name pkg
if db_pkg
db_pkg.visits += count
if visits = packages[db_pkg.name]
db_pkg.visits += visits
db_pkg.save!
Rails.logger.debug "Package: #{db_pkg.name} Visits: #{visits}"
else
Rails.logger.debug "Package #{pkg} not found in database."
Rails.logger.debug "Package: #{db_pkg} (no visits counted)"
end
end
end
end # /find_in_batches
end # /transaction
Rails.logger.info "Done."
end
end
end # /task
end # /namespace