Christoph Haas
61b64fae25
Unique violations on packages_name_key happen on a statement that cannot logically touch name - log the exact query and a longer backtrace to identify what really fails.
65 lines
2.2 KiB
Ruby
65 lines
2.2 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
# Long rake task bodies are normal; the block-length cop is meant for
|
|
# application code.
|
|
# rubocop:disable Metrics/BlockLength
|
|
namespace :debshots do
|
|
desc 'Compute embedding vectors for all packages based on their description ' \
|
|
'and long description (missing ones only unless FORCE=1)'
|
|
task compute_vectors: :environment do
|
|
logger = Logger.new($stdout)
|
|
logger.level = Logger::INFO
|
|
|
|
force = ENV['FORCE'].present?
|
|
# Packages without any description text cannot be embedded at all -
|
|
# exclude them so they do not fail on every single run.
|
|
base_scope = Package.where("COALESCE(description, '') <> '' OR " \
|
|
"COALESCE(long_description, '') <> ''")
|
|
scope = force ? base_scope : base_scope.where(embedding: nil)
|
|
total = scope.count
|
|
|
|
if total.zero?
|
|
logger.info 'No packages need vector computation.'
|
|
next
|
|
end
|
|
|
|
logger.info "Computing vectors for #{total} packages " \
|
|
"(#{force ? 'recomputing' : 'missing only'}, " \
|
|
"service: #{Vectorizer.service_url})"
|
|
|
|
done = 0
|
|
failed = 0
|
|
|
|
# Remember the most recent SQL statement so failures can be traced
|
|
# to the exact query that triggered them.
|
|
last_sql = nil
|
|
ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
|
|
last_sql = args.last[:sql]
|
|
end
|
|
|
|
scope.find_each do |package|
|
|
begin
|
|
raise ArgumentError, 'no description text' if package.embedding_text.blank?
|
|
|
|
package.update_embedding!
|
|
done += 1
|
|
rescue StandardError => e
|
|
# A single broken package must never stop the whole run - log
|
|
# enough detail to diagnose it and move on.
|
|
failed += 1
|
|
logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}"
|
|
logger.error "Last SQL: #{last_sql}"
|
|
logger.error e.backtrace.first(15).join("\n") if e.backtrace
|
|
end
|
|
|
|
processed = done + failed
|
|
next unless (processed % 100).zero? || processed == total
|
|
|
|
percent = (processed.to_f / total * 100).round(1)
|
|
logger.info "Progress: #{processed}/#{total} (#{percent}%)"
|
|
end
|
|
|
|
logger.info "Done. #{done} vectors computed, #{failed} failures."
|
|
end
|
|
end
|
|
# rubocop:enable Metrics/BlockLength
|