Add semantic package search via pgvector embeddings

Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.

Embeddings can be backfilled with bin/rails debshots:compute_vectors.

Also drops the unused lograge gem from the Gemfile.

lib/tasks/vectorize_packages.rake

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+namespace :debshots do
+  desc 'Compute embedding vectors for all packages based on their description ' \
+       'and long description (missing ones only unless FORCE=1). ' \
+       'Set CONCURRENCY=n to control parallel HTTP requests.'
+  task compute_vectors: :environment do
+    logger = Logger.new($stdout)
+    logger.level = Logger::INFO
+
+    force = ENV['FORCE'].present?
+    concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max
+
+    # Never use more threads than the connection pool has connections.
+    # Keep one connection reserved for the main thread.
+    pool_size = ActiveRecord::Base.connection_pool.size
+    concurrency = [concurrency, pool_size - 1].min
+    concurrency = 0 if concurrency.negative?
+
+    scope = force ? Package.all : Package.where(embedding: nil)
+    total = scope.count
+
+    if total.zero?
+      logger.info 'No packages need vector computation.'
+      next
+    end
+
+    logger.info "Computing vectors for #{total} packages " \
+                "(#{force ? 'recomputing' : 'missing only'}, " \
+                "service: #{Vectorizer.service_url}, " \
+                "concurrency: #{concurrency.zero? ? 1 : concurrency})"
+
+    stats_mutex = Mutex.new
+    stats = { done: 0, failed: 0 }
+
+    process_package = lambda do |package|
+      stats_mutex.synchronize do
+        begin
+          raise ArgumentError, 'no description text' if package.embedding_text.blank?
+
+          package.update_embedding!
+          stats[:done] += 1
+        rescue Vectorizer::Error, ArgumentError => e
+          stats[:failed] += 1
+          logger.error "Failed for package #{package.name}: #{e.message}"
+        end
+
+        processed = stats[:done] + stats[:failed]
+        return unless (processed % 100).zero?
+
+        percent = (processed.to_f / total * 100).round(1)
+        logger.info "Progress: #{processed}/#{total} (#{percent}%)"
+      end
+    end
+
+    if concurrency.zero?
+      # Sequential fallback when the connection pool has no room for workers
+      scope.find_each { |package| process_package.call(package) }
+    else
+      # Feed package IDs into a queue that the worker threads pull from.
+      work_queue = Queue.new
+      scope.pluck(:id).each { |id| work_queue << id }
+      concurrency.times { work_queue << nil } # one poison pill per worker
+
+      workers = Array.new(concurrency) do
+        Thread.new do
+          # Each thread gets its own database connection from the pool.
+          ActiveRecord::Base.connection_pool.with_connection do
+            while (package_id = work_queue.pop)
+              package = Package.find_by(id: package_id)
+              process_package.call(package) if package
+            end
+          end
+        rescue StandardError => e
+          logger.error "Worker crashed: #{e.message}"
+        end
+      end
+
+      workers.each(&:join)
+    end
+
+    logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures."
+  end
+end

lib/vectorizer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+# Client for the external web service that turns text into embedding
+# vectors for nearest neighbor search (pgvector).
+#
+#   Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...]
+#
+# The service URL can be overridden with the VECTOR_SERVICE_URL
+# environment variable.
+class Vectorizer
+  # Raised when the vector service cannot be reached or returns garbage
+  class Error < StandardError; end
+
+  DEFAULT_URL = 'https://vector-search.workaround.org/vector'
+
+  class << self
+    # Compute the embedding vector for the given text.
+    # Returns an array of floats.
+    def embed(text)
+      new.embed(text)
+    end
+
+    def service_url
+      ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL)
+    end
+  end
+
+  def embed(text)
+    response = post_text(text)
+
+    unless response.is_a?(Net::HTTPSuccess)
+      raise Error, "Vector service returned HTTP #{response.code}"
+    end
+
+    JSON.parse(response.body).fetch('vector')
+  rescue JSON::ParserError => e
+    raise Error, "Vector service returned invalid JSON: #{e.message}"
+  rescue KeyError
+    raise Error, 'Vector service response did not contain a "vector" field'
+  end
+
+  private
+
+  def post_text(text)
+    uri = URI.parse(self.class.service_url)
+    request = build_request(uri, text)
+
+    options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 }
+    Net::HTTP.start(uri.host, uri.port, **options) do |http|
+      http.request(request)
+    end
+  rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e
+    raise Error, "Could not reach vector service: #{e.message}"
+  end
+
+  def build_request(uri, text)
+    request = Net::HTTP::Post.new(uri.request_uri)
+    request['Content-Type'] = 'application/json'
+    request.body = { text: text }.to_json
+
+    # Optional Bearer token if the service runs with API_SECRET set
+    secret = ENV['VECTOR_SERVICE_SECRET']
+    request['Authorization'] = "Bearer #{secret}" if secret.present?
+
+    request
+  end
+end