Add semantic package search via pgvector embeddings

Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.

Embeddings can be backfilled with bin/rails debshots:compute_vectors.

Also drops the unused lograge gem from the Gemfile.
This commit is contained in:
Christoph Haas 2026-08-22 22:07:56 +02:00
parent c7e8109bbd
commit 4948fe50e3
15 changed files with 662 additions and 125 deletions

View file

@ -0,0 +1,84 @@
# frozen_string_literal: true
namespace :debshots do
desc 'Compute embedding vectors for all packages based on their description ' \
'and long description (missing ones only unless FORCE=1). ' \
'Set CONCURRENCY=n to control parallel HTTP requests.'
task compute_vectors: :environment do
logger = Logger.new($stdout)
logger.level = Logger::INFO
force = ENV['FORCE'].present?
concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max
# Never use more threads than the connection pool has connections.
# Keep one connection reserved for the main thread.
pool_size = ActiveRecord::Base.connection_pool.size
concurrency = [concurrency, pool_size - 1].min
concurrency = 0 if concurrency.negative?
scope = force ? Package.all : Package.where(embedding: nil)
total = scope.count
if total.zero?
logger.info 'No packages need vector computation.'
next
end
logger.info "Computing vectors for #{total} packages " \
"(#{force ? 'recomputing' : 'missing only'}, " \
"service: #{Vectorizer.service_url}, " \
"concurrency: #{concurrency.zero? ? 1 : concurrency})"
stats_mutex = Mutex.new
stats = { done: 0, failed: 0 }
process_package = lambda do |package|
stats_mutex.synchronize do
begin
raise ArgumentError, 'no description text' if package.embedding_text.blank?
package.update_embedding!
stats[:done] += 1
rescue Vectorizer::Error, ArgumentError => e
stats[:failed] += 1
logger.error "Failed for package #{package.name}: #{e.message}"
end
processed = stats[:done] + stats[:failed]
return unless (processed % 100).zero?
percent = (processed.to_f / total * 100).round(1)
logger.info "Progress: #{processed}/#{total} (#{percent}%)"
end
end
if concurrency.zero?
# Sequential fallback when the connection pool has no room for workers
scope.find_each { |package| process_package.call(package) }
else
# Feed package IDs into a queue that the worker threads pull from.
work_queue = Queue.new
scope.pluck(:id).each { |id| work_queue << id }
concurrency.times { work_queue << nil } # one poison pill per worker
workers = Array.new(concurrency) do
Thread.new do
# Each thread gets its own database connection from the pool.
ActiveRecord::Base.connection_pool.with_connection do
while (package_id = work_queue.pop)
package = Package.find_by(id: package_id)
process_package.call(package) if package
end
end
rescue StandardError => e
logger.error "Worker crashed: #{e.message}"
end
end
workers.each(&:join)
end
logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures."
end
end