Add semantic package search via pgvector embeddings

Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.

Embeddings can be backfilled with bin/rails debshots:compute_vectors.

Also drops the unused lograge gem from the Gemfile.

app/controllers/packages_controller.rb

@@ -362,7 +362,7 @@ class PackagesController < ApplicationController
     # text search
     if params[:search].present?
       logger.debug "Searching for #{params[:search]}"
-      packages = packages.general_search(params[:search])
+      packages = search_packages(params[:search])
     end
 
     case params[:show]
@@ -378,6 +378,51 @@ class PackagesController < ApplicationController
     packages
   end
 
+  # Text search for packages: an exact package name match wins without
+  # touching the vector service. Otherwise packages whose name starts
+  # with the query (or contains all query tokens, catching compound
+  # words like "sqlite browser" -> "sqlitebrowser") are promoted above
+  # the semantic (vector) nearest neighbor results. Classic PostgreSQL
+  # full-text search kicks in when the vector service cannot be reached.
+  MAX_PROMOTED_NAME_MATCHES = 8
+  MAX_SEMANTIC_RESULTS = 100
+  private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS
+
+  def search_packages(query)
+    exact_match = Package.find_by(name: query)
+    return Package.where(id: exact_match.id) if exact_match
+
+    tokens = query.to_s.split(/\s+/)
+    name_matches = (
+      Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) +
+      Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES)
+    ).uniq(&:id)
+
+    results = (name_matches + semantic_results(query)).uniq(&:id)
+
+    # Queries that share no vocabulary with our package data can produce
+    # no meaningful results. Skip the vector service round-trip and show
+    # an honest empty result instead of random packages.
+    results.empty? ? Package.none : results
+  end
+
+  # Semantic (vector) nearest neighbor search with a full-text fallback
+  # for when the external embedding service is unreachable.
+  def semantic_results(query)
+    unless Package.lexical_overlap?(query)
+      logger.debug "Query '#{query}' has no lexical overlap with package data"
+      return []
+    end
+
+    begin
+      Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a
+    rescue Vectorizer::Error => e
+      logger.error "Semantic search failed: #{e.message}"
+      flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.'
+      Package.general_search(query).limit(100).to_a
+    end
+  end
+
   # Store a random identifier and the client's IP address in the session
   # for later identification.
   # def create_user_token

app/models/package.rb

@@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open()
 require 'json'
 require 'deb_importer'
 
+# Search and embedding logic keeps accumulating here; the class is
+# cohesive enough that splitting it is not worth the indirection.
+# rubocop:disable Metrics/ClassLength
 class Package < ApplicationRecord
   # PostgreSQL-based full-text search:
   # https://github.com/Casecommons/pg_search
@@ -26,6 +29,11 @@ class Package < ApplicationRecord
            inverse_of: :package,
            dependent: :destroy
 
+  # Nearest neighbor search using pgvector. The "embedding" column stores
+  # a vector computed from description + long_description by an external
+  # web service (see Vectorizer and lib/tasks/vectorize_packages.rake).
+  has_neighbors :embedding
+
   # default_scope {
   # order('name ASC')
   # }
@@ -40,6 +48,94 @@ class Package < ApplicationRecord
     long_description.split(/\n\.\n/).first if long_description
   end
 
+  # The text that the embedding vector is computed from: description
+  # and long description combined.
+  def embedding_text
+    [description, long_description].compact.join("\n")
+  end
+
+  # Compute and store the embedding vector for this package by asking
+  # the external vector service. Raises Vectorizer::Error on failure.
+  # Returns true if the vector was saved.
+  def update_embedding!
+    text = embedding_text
+    return false if text.blank?
+
+    self.embedding = Vectorizer.embed(text)
+    save!
+  end
+
+  # Return a relation of packages whose stored embeddings are closest to
+  # the given query vector (an array of floats). Records get a
+  # "neighbor_distance" attribute added (cosine distance, lower is better).
+  # The returned relation can still be chained (e.g. filtered or paginated).
+  #
+  # Only returns packages that already have an embedding computed.
+  def self.nearest_to_vector(vector, limit: nil)
+    neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine')
+    limit ? neighbors.limit(limit) : neighbors
+  end
+
+  # Turn a free-text search string into a vector via the external web
+  # service and find the closest packages.
+  def self.nearest_to_text(text, limit: nil)
+    return none if text.blank?
+
+    nearest_to_vector(Vectorizer.embed(text), limit: limit)
+  end
+
+  # Packages whose name starts with the given string - used to promote
+  # obvious matches like "sqlite3" for a "sqlite" search.
+  def self.name_starts_with(text)
+    return none if text.blank?
+
+    where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name)
+  end
+
+  # Packages whose name contains all given tokens as substrings. Catches
+  # compound words split by the user, e.g. "sqlite browser" finding the
+  # "sqlitebrowser" package.
+  def self.name_contains_all(tokens)
+    tokens = tokens.select { |token| token.length >= 2 }
+    return none if tokens.empty?
+
+    scope = self
+    tokens.each do |token|
+      scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\')
+    end
+    scope.order(:name)
+  end
+
+  def self.escape_for_like(text)
+    # Escape LIKE wildcards so that package name characters like "+" or "."
+    # cannot break the pattern
+    text.gsub(/[\\%_]/) { |char| "\\#{char}" }
+  end
+
+  # Check whether the given text shares at least one lexeme with any
+  # package name/description in the database. Queries without such an
+  # overlap (gibberish, keyboard mash, unsupported languages) can
+  # neither match full-text nor produce meaningful semantic results -
+  # the embedding model maps unknown tokens close to the corpus
+  # centroid, making them look like good matches.
+  def self.lexical_overlap?(text)
+    tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) }
+    return false if tokens.empty?
+
+    # Pin the same text search dictionary that the packages_fts index
+    # uses - the server-wide default can be anything.
+    tsquery = tokens.map do |token|
+      sanitize_sql(["plainto_tsquery('english', ?)", token])
+    end.join(' || ')
+    # The expression must match the packages_fts GIN index definition
+    # so that Postgres can use the index.
+    where(
+      "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \
+      "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \
+      "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})"
+    ).exists?
+  end
+
   # Return a query of all packages that have screenshots
   def self.with_screenshots
     # Query for all packages who's ID appears in a screenshot's "package_id" field

app/views/packages/browse.slim

@@ -14,7 +14,7 @@
     - if @packages.any?
       - if @view_style==:grid
         .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium"
-          - @packages.all.each do |pkg|
+          - @packages.each do |pkg|
             .cell.pkgcard data-equalizer-watch=true
               a.black href=package_path(name: pkg.name)
                 .image