Christoph Haas
2b79470feb
Drop the threaded worker pool (queue, mutexes, connection pooling). The task now processes packages one by one via find_each - plenty fast since the external embedding service dominates latency, and much easier to reason about.
53 lines
1.7 KiB
Ruby
53 lines
1.7 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
# Long rake task bodies are normal; the block-length cop is meant for
|
|
# application code.
|
|
# rubocop:disable Metrics/BlockLength
|
|
namespace :debshots do
|
|
desc 'Compute embedding vectors for all packages based on their description ' \
|
|
'and long description (missing ones only unless FORCE=1)'
|
|
task compute_vectors: :environment do
|
|
logger = Logger.new($stdout)
|
|
logger.level = Logger::INFO
|
|
|
|
force = ENV['FORCE'].present?
|
|
scope = force ? Package.all : Package.where(embedding: nil)
|
|
total = scope.count
|
|
|
|
if total.zero?
|
|
logger.info 'No packages need vector computation.'
|
|
next
|
|
end
|
|
|
|
logger.info "Computing vectors for #{total} packages " \
|
|
"(#{force ? 'recomputing' : 'missing only'}, " \
|
|
"service: #{Vectorizer.service_url})"
|
|
|
|
done = 0
|
|
failed = 0
|
|
|
|
scope.find_each do |package|
|
|
begin
|
|
raise ArgumentError, 'no description text' if package.embedding_text.blank?
|
|
|
|
package.update_embedding!
|
|
done += 1
|
|
rescue StandardError => e
|
|
# A single broken package must never stop the whole run - log
|
|
# enough detail to diagnose it and move on.
|
|
failed += 1
|
|
logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}"
|
|
logger.error e.backtrace.first(3).join("\n") if e.backtrace
|
|
end
|
|
|
|
processed = done + failed
|
|
next unless (processed % 100).zero? || processed == total
|
|
|
|
percent = (processed.to_f / total * 100).round(1)
|
|
logger.info "Progress: #{processed}/#{total} (#{percent}%)"
|
|
end
|
|
|
|
logger.info "Done. #{done} vectors computed, #{failed} failures."
|
|
end
|
|
end
|
|
# rubocop:enable Metrics/BlockLength
|