Harden vectorization task against per-package failures
A single failing package used to kill its whole worker thread silently because only Vectorizer::Error was rescued per package. Now every StandardError is caught and logged with class and backtrace, and the embedding write is narrowed to update_column(:embedding) so the task only ever issues a minimal UPDATE - it cannot touch any other column or fire model callbacks. Progress stats no longer serialize database writes through the mutex.
This commit is contained in:
parent
4d07f43a2e
commit
3550a4e2e2
2 changed files with 18 additions and 13 deletions
|
|
@ -56,13 +56,12 @@ class Package < ApplicationRecord
|
|||
|
||||
# Compute and store the embedding vector for this package by asking
|
||||
# the external vector service. Raises Vectorizer::Error on failure.
|
||||
# Returns true if the vector was saved.
|
||||
# Returns true if a vector was saved.
|
||||
def update_embedding!
|
||||
text = embedding_text
|
||||
return false if text.blank?
|
||||
|
||||
self.embedding = Vectorizer.embed(text)
|
||||
save!
|
||||
update_column(:embedding, Vectorizer.embed(text))
|
||||
end
|
||||
|
||||
# Return a relation of packages whose stored embeddings are closest to
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
# Long rake task bodies are normal; the block-length cop is meant for
|
||||
# application code.
|
||||
# rubocop:disable Metrics/BlockLength
|
||||
namespace :debshots do
|
||||
desc 'Compute embedding vectors for all packages based on their description ' \
|
||||
'and long description (missing ones only unless FORCE=1). ' \
|
||||
|
|
@ -34,17 +37,20 @@ namespace :debshots do
|
|||
stats = { done: 0, failed: 0 }
|
||||
|
||||
process_package = lambda do |package|
|
||||
stats_mutex.synchronize do
|
||||
begin
|
||||
raise ArgumentError, 'no description text' if package.embedding_text.blank?
|
||||
|
||||
package.update_embedding!
|
||||
stats[:done] += 1
|
||||
rescue Vectorizer::Error, ArgumentError => e
|
||||
stats[:failed] += 1
|
||||
logger.error "Failed for package #{package.name}: #{e.message}"
|
||||
stats_mutex.synchronize { stats[:done] += 1 }
|
||||
rescue StandardError => e
|
||||
# A single broken package must never take down a whole worker
|
||||
# thread - log enough detail to diagnose it and move on.
|
||||
stats_mutex.synchronize { stats[:failed] += 1 }
|
||||
logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}"
|
||||
logger.error e.backtrace.first(3).join("\n") if e.backtrace
|
||||
end
|
||||
|
||||
stats_mutex.synchronize do
|
||||
processed = stats[:done] + stats[:failed]
|
||||
return unless (processed % 100).zero?
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue