Add semantic package search via pgvector embeddings

Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.

Embeddings can be backfilled with bin/rails debshots:compute_vectors.

Also drops the unused lograge gem from the Gemfile.

Gemfile

@@ -41,8 +41,7 @@ gem 'mini_magick'
 # Reduces boot times through caching; required in config/boot.rb
 gem 'bootsnap', require: false
 
-# Structured logging for production
+
-gem 'lograge'
 
 # Deploy this application anywhere as a Docker container [https://kamal-deploy.org]
 gem 'kamal', require: false
@@ -124,6 +123,10 @@ gem 'will_paginate-foundation'
 # Full-text search in PostgreSQL
 gem 'pg_search'
 
+# Nearest neighbor search for PostgreSQL using the pgvector extension
+# https://github.com/ankane/neighbor
+gem 'neighbor'
+
 # Use SLIM as our templating language
 gem 'slim-rails'
 

Gemfile.lock

@@ -224,11 +224,6 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.7.0)
-    lograge (0.14.0)
-      actionpack (>= 4)
-      activesupport (>= 4)
-      railties (>= 4)
-      request_store (~> 1.0)
     loofah (2.25.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -255,6 +250,8 @@ GEM
       minitest (>= 5.0, < 7)
       ruby-progressbar
     msgpack (1.8.0)
+    neighbor (1.2.0)
+      activerecord (>= 7.2)
     nenv (0.3.0)
     net-http (0.9.1)
       uri (>= 0.11.1)
@@ -427,8 +424,6 @@ GEM
     regexp_parser (2.12.0)
     reline (0.6.3)
       io-console (~> 0.5)
-    request_store (1.7.0)
-      rack (>= 1.4)
     responders (3.2.0)
       actionpack (>= 7.0)
       railties (>= 7.0)
@@ -585,10 +580,10 @@ DEPENDENCIES
   jbuilder
   kamal
   listen (~> 3.5)
-  lograge
   mini_magick
   minitest-rails
   minitest-reporters
+  neighbor
   omniauth
   omniauth-rails_csrf_protection
   omniauth_openid_connect

README.md

@@ -15,6 +15,30 @@ https://salsa.debian.org/debian/debshots/-/issues
 
 Check out the doc/README.Development.md
 
+## Package search
+
+The package search combines several strategies, tried in this order:
+
+1. **Exact package name**: Searching for `vim` shows exactly this
+   package without calling any external service.
+2. **Name prefix matches**: Packages whose name starts with the query
+   rank first, e.g. searching for `sqlite` promotes `sqlite3`,
+   `sqlitebrowser` and friends above all other results.
+3. **Compound word splitting**: Packages whose name contains *all*
+   query tokens match too, so `sqlite browser` finds `sqlitebrowser`.
+4. **Semantic search**: The query is embedded via the vector service
+   (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among
+   the package description embeddings are returned, using the pgvector
+   extension with an HNSW cosine index. Embeddings are computed from
+   the package description and long description and can be (re-)built
+   with `bin/rails debshots:compute_vectors`.
+5. **Full-text fallback**: If the vector service cannot be reached,
+   classic PostgreSQL full-text search takes over.
+
+Queries that share no vocabulary with the package data (gibberish,
+keyboard mash) return an honest empty result instead of random
+semantic matches.
+
 ## Deployment
 
 Read the doc/README.Installation.md

app/controllers/packages_controller.rb

@@ -362,7 +362,7 @@ class PackagesController < ApplicationController
     # text search
     if params[:search].present?
       logger.debug "Searching for #{params[:search]}"
-      packages = packages.general_search(params[:search])
+      packages = search_packages(params[:search])
     end
 
     case params[:show]
@@ -378,6 +378,51 @@ class PackagesController < ApplicationController
     packages
   end
 
+  # Text search for packages: an exact package name match wins without
+  # touching the vector service. Otherwise packages whose name starts
+  # with the query (or contains all query tokens, catching compound
+  # words like "sqlite browser" -> "sqlitebrowser") are promoted above
+  # the semantic (vector) nearest neighbor results. Classic PostgreSQL
+  # full-text search kicks in when the vector service cannot be reached.
+  MAX_PROMOTED_NAME_MATCHES = 8
+  MAX_SEMANTIC_RESULTS = 100
+  private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS
+
+  def search_packages(query)
+    exact_match = Package.find_by(name: query)
+    return Package.where(id: exact_match.id) if exact_match
+
+    tokens = query.to_s.split(/\s+/)
+    name_matches = (
+      Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) +
+      Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES)
+    ).uniq(&:id)
+
+    results = (name_matches + semantic_results(query)).uniq(&:id)
+
+    # Queries that share no vocabulary with our package data can produce
+    # no meaningful results. Skip the vector service round-trip and show
+    # an honest empty result instead of random packages.
+    results.empty? ? Package.none : results
+  end
+
+  # Semantic (vector) nearest neighbor search with a full-text fallback
+  # for when the external embedding service is unreachable.
+  def semantic_results(query)
+    unless Package.lexical_overlap?(query)
+      logger.debug "Query '#{query}' has no lexical overlap with package data"
+      return []
+    end
+
+    begin
+      Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a
+    rescue Vectorizer::Error => e
+      logger.error "Semantic search failed: #{e.message}"
+      flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.'
+      Package.general_search(query).limit(100).to_a
+    end
+  end
+
   # Store a random identifier and the client's IP address in the session
   # for later identification.
   # def create_user_token

app/models/package.rb

@@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open()
 require 'json'
 require 'deb_importer'
 
+# Search and embedding logic keeps accumulating here; the class is
+# cohesive enough that splitting it is not worth the indirection.
+# rubocop:disable Metrics/ClassLength
 class Package < ApplicationRecord
   # PostgreSQL-based full-text search:
   # https://github.com/Casecommons/pg_search
@@ -26,6 +29,11 @@ class Package < ApplicationRecord
            inverse_of: :package,
            dependent: :destroy
 
+  # Nearest neighbor search using pgvector. The "embedding" column stores
+  # a vector computed from description + long_description by an external
+  # web service (see Vectorizer and lib/tasks/vectorize_packages.rake).
+  has_neighbors :embedding
+
   # default_scope {
   # order('name ASC')
   # }
@@ -40,6 +48,94 @@ class Package < ApplicationRecord
     long_description.split(/\n\.\n/).first if long_description
   end
 
+  # The text that the embedding vector is computed from: description
+  # and long description combined.
+  def embedding_text
+    [description, long_description].compact.join("\n")
+  end
+
+  # Compute and store the embedding vector for this package by asking
+  # the external vector service. Raises Vectorizer::Error on failure.
+  # Returns true if the vector was saved.
+  def update_embedding!
+    text = embedding_text
+    return false if text.blank?
+
+    self.embedding = Vectorizer.embed(text)
+    save!
+  end
+
+  # Return a relation of packages whose stored embeddings are closest to
+  # the given query vector (an array of floats). Records get a
+  # "neighbor_distance" attribute added (cosine distance, lower is better).
+  # The returned relation can still be chained (e.g. filtered or paginated).
+  #
+  # Only returns packages that already have an embedding computed.
+  def self.nearest_to_vector(vector, limit: nil)
+    neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine')
+    limit ? neighbors.limit(limit) : neighbors
+  end
+
+  # Turn a free-text search string into a vector via the external web
+  # service and find the closest packages.
+  def self.nearest_to_text(text, limit: nil)
+    return none if text.blank?
+
+    nearest_to_vector(Vectorizer.embed(text), limit: limit)
+  end
+
+  # Packages whose name starts with the given string - used to promote
+  # obvious matches like "sqlite3" for a "sqlite" search.
+  def self.name_starts_with(text)
+    return none if text.blank?
+
+    where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name)
+  end
+
+  # Packages whose name contains all given tokens as substrings. Catches
+  # compound words split by the user, e.g. "sqlite browser" finding the
+  # "sqlitebrowser" package.
+  def self.name_contains_all(tokens)
+    tokens = tokens.select { |token| token.length >= 2 }
+    return none if tokens.empty?
+
+    scope = self
+    tokens.each do |token|
+      scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\')
+    end
+    scope.order(:name)
+  end
+
+  def self.escape_for_like(text)
+    # Escape LIKE wildcards so that package name characters like "+" or "."
+    # cannot break the pattern
+    text.gsub(/[\\%_]/) { |char| "\\#{char}" }
+  end
+
+  # Check whether the given text shares at least one lexeme with any
+  # package name/description in the database. Queries without such an
+  # overlap (gibberish, keyboard mash, unsupported languages) can
+  # neither match full-text nor produce meaningful semantic results -
+  # the embedding model maps unknown tokens close to the corpus
+  # centroid, making them look like good matches.
+  def self.lexical_overlap?(text)
+    tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) }
+    return false if tokens.empty?
+
+    # Pin the same text search dictionary that the packages_fts index
+    # uses - the server-wide default can be anything.
+    tsquery = tokens.map do |token|
+      sanitize_sql(["plainto_tsquery('english', ?)", token])
+    end.join(' || ')
+    # The expression must match the packages_fts GIN index definition
+    # so that Postgres can use the index.
+    where(
+      "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \
+      "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \
+      "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})"
+    ).exists?
+  end
+
   # Return a query of all packages that have screenshots
   def self.with_screenshots
     # Query for all packages who's ID appears in a screenshot's "package_id" field

app/views/packages/browse.slim

@@ -14,7 +14,7 @@
     - if @packages.any?
       - if @view_style==:grid
         .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium"
-          - @packages.all.each do |pkg|
+          - @packages.each do |pkg|
             .cell.pkgcard data-equalizer-watch=true
               a.black href=package_path(name: pkg.name)
                 .image

config/initializers/will_paginate.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Enable will_paginate's Array#paginate so that merged search results
+# (name prefix matches + semantic results) can be paginated like
+# ActiveRecord relations.
+require 'will_paginate/array'

db/migrate/20260821203833_install_neighbor_vector.rb

@@ -0,0 +1,5 @@
+class InstallNeighborVector < ActiveRecord::Migration[8.1]
+  def change
+    enable_extension "vector"
+  end
+end

db/migrate/20260821210000_add_embedding_to_packages.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Adds a vector column to store text embeddings of the package
+# description and long description. The vectors are computed by an
+# external web service and used for nearest neighbor search.
+#
+# The pgvector extension is enabled by the InstallNeighborVector migration.
+class AddEmbeddingToPackages < ActiveRecord::Migration[8.1]
+  def change
+    add_column :packages, :embedding, :vector, limit: 384
+
+    # HNSW index for fast approximate nearest neighbor searches
+    add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops
+  end
+end

db/schema.rb

@@ -10,133 +10,131 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do
+ActiveRecord::Schema[8.1].define(version: 2026_08_21_210000) do
-  # These are extensions that must be enabled to support this database
+  # These are extensions that must be enabled in order to support this database
-  enable_extension 'pg_catalog.plpgsql'
+  enable_extension "pg_catalog.plpgsql"
+  enable_extension "vector"
 
-  create_table 'action_mailbox_inbound_emails', force: :cascade do |t|
+  create_table "action_mailbox_inbound_emails", force: :cascade do |t|
-    t.integer 'status', default: 0, null: false
+    t.datetime "created_at", null: false
-    t.string 'message_id', null: false
+    t.string "message_checksum", null: false
-    t.string 'message_checksum', null: false
+    t.string "message_id", null: false
-    t.datetime 'created_at', null: false
+    t.integer "status", default: 0, null: false
-    t.datetime 'updated_at', null: false
+    t.datetime "updated_at", null: false
-    t.index ['message_id', 'message_checksum'], 
+    t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true
-name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true
   end
 
-  create_table 'action_text_rich_texts', force: :cascade do |t|
+  create_table "action_text_rich_texts", force: :cascade do |t|
-    t.string 'name', null: false
+    t.text "body"
-    t.text 'body'
+    t.datetime "created_at", null: false
-    t.string 'record_type', null: false
+    t.string "name", null: false
-    t.bigint 'record_id', null: false
+    t.bigint "record_id", null: false
-    t.datetime 'created_at', null: false
+    t.string "record_type", null: false
-    t.datetime 'updated_at', null: false
+    t.datetime "updated_at", null: false
-    t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', 
+    t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true
-unique: true
   end
 
-  create_table 'active_storage_attachments', force: :cascade do |t|
+  create_table "active_storage_attachments", force: :cascade do |t|
-    t.string 'name', null: false
+    t.bigint "blob_id", null: false
-    t.string 'record_type', null: false
+    t.datetime "created_at", precision: nil, null: false
-    t.bigint 'record_id', null: false
+    t.string "name", null: false
-    t.bigint 'blob_id', null: false
+    t.bigint "record_id", null: false
-    t.datetime 'created_at', precision: nil, null: false
+    t.string "record_type", null: false
-    t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id'
+    t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id"
-    t.index ['record_type', 'record_id', 'name', 'blob_id'], 
+    t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true
-name: 'index_active_storage_attachments_uniqueness', unique: true
   end
 
-  create_table 'active_storage_blobs', force: :cascade do |t|
+  create_table "active_storage_blobs", force: :cascade do |t|
-    t.string 'key', null: false
+    t.bigint "byte_size", null: false
-    t.string 'filename', null: false
+    t.string "checksum"
-    t.string 'content_type'
+    t.string "content_type"
-    t.text 'metadata'
+    t.datetime "created_at", precision: nil, null: false
-    t.bigint 'byte_size', null: false
+    t.string "filename", null: false
-    t.string 'checksum'
+    t.string "key", null: false
-    t.datetime 'created_at', precision: nil, null: false
+    t.text "metadata"
-    t.string 'service_name', null: false
+    t.string "service_name", null: false
-    t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true
+    t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true
   end
 
-  create_table 'active_storage_variant_records', force: :cascade do |t|
+  create_table "active_storage_variant_records", force: :cascade do |t|
-    t.bigint 'blob_id', null: false
+    t.bigint "blob_id", null: false
-    t.string 'variation_digest', null: false
+    t.string "variation_digest", null: false
-    t.index ['blob_id', 'variation_digest'], 
+    t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true
-name: 'index_active_storage_variant_records_uniqueness', unique: true
   end
 
-  create_table 'logs', id: :serial, force: :cascade do |t|
+  create_table "logs", id: :serial, force: :cascade do |t|
-    t.string 'message'
+    t.datetime "created_at", precision: nil, null: false
-    t.string 'level'
+    t.inet "ip_address"
-    t.string 'section'
+    t.string "level"
-    t.inet 'ip_address'
+    t.string "message"
-    t.datetime 'created_at', precision: nil, null: false
+    t.integer "package_id"
-    t.datetime 'updated_at', precision: nil, null: false
+    t.integer "screenshot_id"
-    t.integer 'user_id'
+    t.string "section"
-    t.integer 'package_id'
+    t.datetime "updated_at", precision: nil, null: false
-    t.integer 'screenshot_id'
+    t.integer "user_id"
   end
 
-  create_table 'packages', id: :serial, force: :cascade do |t|
+  create_table "packages", id: :serial, force: :cascade do |t|
-    t.string 'name', null: false
+    t.datetime "created_at", precision: nil
-    t.string 'description', limit: 80
+    t.string "description", limit: 80
-    t.string 'section', limit: 50
+    t.vector "embedding", limit: 384
-    t.string 'maintainer', limit: 100
+    t.string "homepage", limit: 400
-    t.string 'maintainer_email', limit: 100
+    t.text "long_description"
-    t.string 'homepage', limit: 400
+    t.string "maintainer", limit: 100
-    t.string 'version', limit: 200
+    t.string "maintainer_email", limit: 100
-    t.text 'long_description'
+    t.string "name", null: false
-    t.string 'origin', limit: 80
+    t.string "origin", limit: 80
-    t.datetime 'created_at', precision: nil
+    t.string "section", limit: 50
-    t.datetime 'updated_at', precision: nil
+    t.datetime "updated_at", precision: nil
-    t.integer 'visits', default: 0
+    t.string "version", limit: 200
-    t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", 
+    t.integer "visits", default: 0
-name: 'packages_fts', using: :gin
+    t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin
-    t.unique_constraint ['name'], name: 'packages_name_key'
+    t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw
+    t.unique_constraint ["name"], name: "packages_name_key"
   end
 
-  create_table 'screenshots', id: :serial, force: :cascade do |t|
+  create_table "screenshots", id: :serial, force: :cascade do |t|
-    t.integer 'package_id'
+    t.boolean "approved", default: false, null: false
-    t.string 'version', limit: 50
+    t.datetime "created_at", precision: nil
-    t.datetime 'created_at', precision: nil
+    t.text "description"
-    t.string 'uploaderhash', limit: 72
+    t.boolean "hidden", default: false
-    t.boolean 'approved', default: false, null: false
+    t.string "image_fingerprint"
-    t.text 'description'
+    t.integer "package_id"
-    t.datetime 'updated_at', precision: nil
+    t.text "simage_data"
-    t.string 'image_fingerprint'
+    t.datetime "updated_at", precision: nil
-    t.integer 'user_id', default: 0
+    t.string "uploaderhash", limit: 72
-    t.text 'simage_data'
+    t.inet "uploaderip"
-    t.inet 'uploaderip'
+    t.integer "user_id", default: 0
-    t.boolean 'hidden', default: false
+    t.string "version", limit: 50
-    t.index ['id', 'approved'], name: 'id_approved'
+    t.index ["id", "approved"], name: "id_approved"
-    t.index ['id', 'uploaderhash'], name: 'id_uploaderhash'
+    t.index ["id", "uploaderhash"], name: "id_uploaderhash"
   end
 
-  create_table 'users', id: :serial, force: :cascade do |t|
+  create_table "users", id: :serial, force: :cascade do |t|
-    t.text 'name'
+    t.boolean "admin_role", default: false
-    t.datetime 'created_at', precision: nil, null: false
+    t.integer "approved_screenshots", default: 0
-    t.datetime 'updated_at', precision: nil, null: false
+    t.datetime "created_at", precision: nil, null: false
-    t.string 'email', default: '', null: false
+    t.datetime "current_sign_in_at", precision: nil
-    t.string 'encrypted_password', default: '', null: false
+    t.inet "current_sign_in_ip"
-    t.integer 'sign_in_count', default: 0, null: false
+    t.string "email", default: "", null: false
-    t.datetime 'current_sign_in_at', precision: nil
+    t.string "encrypted_password", default: "", null: false
-    t.datetime 'last_sign_in_at', precision: nil
+    t.integer "failed_attempts", default: 0, null: false
-    t.inet 'current_sign_in_ip'
+    t.datetime "last_sign_in_at", precision: nil
-    t.inet 'last_sign_in_ip'
+    t.inet "last_sign_in_ip"
-    t.integer 'failed_attempts', default: 0, null: false
+    t.datetime "locked_at", precision: nil
-    t.string 'unlock_token'
+    t.boolean "moderator_role", default: false
-    t.datetime 'locked_at', precision: nil
+    t.text "name"
-    t.string 'provider'
+    t.string "provider"
-    t.string 'uid'
+    t.boolean "pseudo", default: false
-    t.boolean 'admin_role', default: false
+    t.integer "rejected_screenshots", default: 0
-    t.boolean 'moderator_role', default: false
+    t.integer "sign_in_count", default: 0, null: false
-    t.boolean 'pseudo', default: false
+    t.string "uid"
-    t.integer 'approved_screenshots', default: 0
+    t.string "unlock_token"
-    t.integer 'rejected_screenshots', default: 0
+    t.datetime "updated_at", precision: nil, null: false
-    t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true
+    t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true
   end
 
-  add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id'
+  add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
-  add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id'
+  add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
-  add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey'
+  add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey"
 end

lib/tasks/vectorize_packages.rake

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+namespace :debshots do
+  desc 'Compute embedding vectors for all packages based on their description ' \
+       'and long description (missing ones only unless FORCE=1). ' \
+       'Set CONCURRENCY=n to control parallel HTTP requests.'
+  task compute_vectors: :environment do
+    logger = Logger.new($stdout)
+    logger.level = Logger::INFO
+
+    force = ENV['FORCE'].present?
+    concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max
+
+    # Never use more threads than the connection pool has connections.
+    # Keep one connection reserved for the main thread.
+    pool_size = ActiveRecord::Base.connection_pool.size
+    concurrency = [concurrency, pool_size - 1].min
+    concurrency = 0 if concurrency.negative?
+
+    scope = force ? Package.all : Package.where(embedding: nil)
+    total = scope.count
+
+    if total.zero?
+      logger.info 'No packages need vector computation.'
+      next
+    end
+
+    logger.info "Computing vectors for #{total} packages " \
+                "(#{force ? 'recomputing' : 'missing only'}, " \
+                "service: #{Vectorizer.service_url}, " \
+                "concurrency: #{concurrency.zero? ? 1 : concurrency})"
+
+    stats_mutex = Mutex.new
+    stats = { done: 0, failed: 0 }
+
+    process_package = lambda do |package|
+      stats_mutex.synchronize do
+        begin
+          raise ArgumentError, 'no description text' if package.embedding_text.blank?
+
+          package.update_embedding!
+          stats[:done] += 1
+        rescue Vectorizer::Error, ArgumentError => e
+          stats[:failed] += 1
+          logger.error "Failed for package #{package.name}: #{e.message}"
+        end
+
+        processed = stats[:done] + stats[:failed]
+        return unless (processed % 100).zero?
+
+        percent = (processed.to_f / total * 100).round(1)
+        logger.info "Progress: #{processed}/#{total} (#{percent}%)"
+      end
+    end
+
+    if concurrency.zero?
+      # Sequential fallback when the connection pool has no room for workers
+      scope.find_each { |package| process_package.call(package) }
+    else
+      # Feed package IDs into a queue that the worker threads pull from.
+      work_queue = Queue.new
+      scope.pluck(:id).each { |id| work_queue << id }
+      concurrency.times { work_queue << nil } # one poison pill per worker
+
+      workers = Array.new(concurrency) do
+        Thread.new do
+          # Each thread gets its own database connection from the pool.
+          ActiveRecord::Base.connection_pool.with_connection do
+            while (package_id = work_queue.pop)
+              package = Package.find_by(id: package_id)
+              process_package.call(package) if package
+            end
+          end
+        rescue StandardError => e
+          logger.error "Worker crashed: #{e.message}"
+        end
+      end
+
+      workers.each(&:join)
+    end
+
+    logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures."
+  end
+end

lib/vectorizer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+# Client for the external web service that turns text into embedding
+# vectors for nearest neighbor search (pgvector).
+#
+#   Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...]
+#
+# The service URL can be overridden with the VECTOR_SERVICE_URL
+# environment variable.
+class Vectorizer
+  # Raised when the vector service cannot be reached or returns garbage
+  class Error < StandardError; end
+
+  DEFAULT_URL = 'https://vector-search.workaround.org/vector'
+
+  class << self
+    # Compute the embedding vector for the given text.
+    # Returns an array of floats.
+    def embed(text)
+      new.embed(text)
+    end
+
+    def service_url
+      ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL)
+    end
+  end
+
+  def embed(text)
+    response = post_text(text)
+
+    unless response.is_a?(Net::HTTPSuccess)
+      raise Error, "Vector service returned HTTP #{response.code}"
+    end
+
+    JSON.parse(response.body).fetch('vector')
+  rescue JSON::ParserError => e
+    raise Error, "Vector service returned invalid JSON: #{e.message}"
+  rescue KeyError
+    raise Error, 'Vector service response did not contain a "vector" field'
+  end
+
+  private
+
+  def post_text(text)
+    uri = URI.parse(self.class.service_url)
+    request = build_request(uri, text)
+
+    options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 }
+    Net::HTTP.start(uri.host, uri.port, **options) do |http|
+      http.request(request)
+    end
+  rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e
+    raise Error, "Could not reach vector service: #{e.message}"
+  end
+
+  def build_request(uri, text)
+    request = Net::HTTP::Post.new(uri.request_uri)
+    request['Content-Type'] = 'application/json'
+    request.body = { text: text }.to_json
+
+    # Optional Bearer token if the service runs with API_SECRET set
+    secret = ENV['VECTOR_SERVICE_SECRET']
+    request['Authorization'] = "Bearer #{secret}" if secret.present?
+
+    request
+  end
+end

test/controllers/packages_controller_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackagesControllerTest < ActionController::TestCase
@@ -20,6 +22,81 @@ class PackagesControllerTest < ActionController::TestCase
     assert_select 'div', 'A program to browse web sites.'
   end
 
+  test 'exact package name search wins without calling the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'vim' }
+      assert_response :success
+      assert_select 'div', 'vim'
+    end
+  end
+
+  test 'gibberish search shows no results and never calls the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'asdfgh zzzz' }
+      assert_response :success
+      # the grid renders every package in the database when no search
+      # filter applies - so an empty result must not fall back to that
+      assert_select 'div', text: /package-\d+/, count: 0
+    end
+  end
+
+  test 'name prefix matches rank above semantic results' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do
+      get :grid, params: { search: 'package' }
+      assert_response :success
+      # fixtures create package-0..package-99 - prefix hits must render
+      assert_select 'div', 'package-0'
+      # ...and before any semantically found package
+      assert response.body.index('package-0') < response.body.index('vim')
+    end
+  end
+
+  test 'prefix matches and semantic results are deduplicated' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do
+      get :grid, params: { search: 'fire' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+    end
+  end
+
+  test 'compound words in search match packages with joined names' do
+    # "fire fox" should find "firefox" via name substring matching
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.none }) do
+      get :grid, params: { search: 'fire fox' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+    end
+  end
+
+  test 'search uses semantic (vector) results' do
+    relation = Package.where(name: packages(:vim).name)
+    with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+    end
+  end
+
+  # There is deliberately no fallback for an empty vector search
+  # result: KNN always returns something as long as packages have
+  # embeddings. Full-text search is only used when the vector service
+  # cannot be reached.
+
+  test 'search falls back to full-text search when vector service fails' do
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { raise Vectorizer::Error, 'service down' }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+      assert flash[:error].present?
+    end
+  end
+
   # test 'should get thumbnail for a package and a desired version' do
   #   # get '/thumbnail-with-version/package-5/0.1'
   #   get thumbnail_with_version_url('package-5', '0.1')

test/models/package_test.rb

@@ -1,7 +1,107 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackageTest < ActiveSupport::TestCase
-  # test "the truth" do
+  test 'embedding_text combines description and long description' do
-  #   assert true
+    package = packages(:firefox)
-  # end
+    assert_equal "A web browser\nA program to browse web sites.", package.embedding_text
+  end
+
+  test 'embedding_text works with missing long description' do
+    package = packages(:firefox)
+    package.long_description = nil
+    assert_equal 'A web browser', package.embedding_text
+  end
+
+  test 'update_embedding! stores the vector returned by the vector service' do
+    vector = Array.new(384) { |i| i / 384.0 }
+    with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do
+      package = packages(:vim)
+      assert package.update_embedding!
+      # The vector type stores float32 so allow small rounding differences
+      package.reload.embedding.to_a.each_with_index do |actual, i|
+        assert_in_delta vector[i], actual, 1e-6
+      end
+    end
+  end
+
+  test 'update_embedding! does nothing without description text' do
+    package = packages(:firefox)
+    package.description = nil
+    package.long_description = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { flunk 'should not call the vector service' }) do
+      assert_not package.update_embedding!
+    end
+  end
+
+  test 'nearest_to_text returns none for blank text' do
+    assert_empty Package.nearest_to_text('')
+    assert_empty Package.nearest_to_text(nil)
+  end
+
+  test 'name_starts_with matches name prefixes case-insensitively' do
+    assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name
+    assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name
+    assert_empty Package.name_starts_with('')
+    assert_empty Package.name_starts_with(nil)
+  end
+
+  test 'name_contains_all finds compound words split by spaces' do
+    assert_includes Package.name_contains_all(%w[fire fox]).map(&:name),
+                    packages(:firefox).name
+    # all tokens must be present in the name
+    assert_empty Package.name_contains_all(%w[firefox zzzz])
+    assert_empty Package.name_contains_all([])
+    # single character tokens would flood results - ignore them
+    assert_empty Package.name_contains_all(['f'])
+  end
+
+  test 'lexical_overlap? pins the english dictionary regardless of server config' do
+    # Regression: with default_text_search_config = german (as on some
+    # servers) 'browser' stems to 'brows' which never matches the
+    # english-stemmed index.
+    connection = ActiveRecord::Base.connection
+    original = connection.select_value('SHOW default_text_search_config')
+    connection.execute("SET default_text_search_config = 'german'")
+    assert Package.lexical_overlap?('browser')
+  ensure
+    connection&.execute("SET default_text_search_config = '#{original}'")
+  end
+
+  test 'lexical_overlap? detects real words vs gibberish' do
+    # words present in fixture package descriptions
+    assert Package.lexical_overlap?('browser')
+    assert Package.lexical_overlap?('email client for vim syntax')
+    assert Package.lexical_overlap?('the browser') # stopwords are ignored
+    # gibberish, numbers and empty strings share no vocabulary
+    assert_not Package.lexical_overlap?('asdfgh zzzz')
+    assert_not Package.lexical_overlap?('123 !!!')
+    assert_not Package.lexical_overlap?('the') # stopword-only query
+    assert_not Package.lexical_overlap?('')
+    assert_not Package.lexical_overlap?(nil)
+  end
+
+  test 'nearest_to_text embeds the query and searches' do
+    vector = Array.new(384, 0.25)
+    embed_called_with = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        lambda { |text|
+                          embed_called_with = text
+                          vector
+                        }) do
+      received = []
+      results = with_stubbed_method(Package, :nearest_neighbors,
+                                    lambda { |_field, received_vector, **_opts|
+                                      received << received_vector
+                                      [packages(:vim)]
+                                    }) do
+        Package.nearest_to_text('some search text')
+      end
+      assert_equal [packages(:vim)], results
+      assert_equal 'some search text', embed_called_with
+      assert_equal [vector], received
+    end
+  end
 end

test/test_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ENV['RAILS_ENV'] ||= 'test'
 require File.expand_path('../config/environment', __dir__)
 require 'rails/test_help'
@@ -15,6 +17,23 @@ module ActiveSupport
   class TestCase
     # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
     fixtures :all
+
+    # Replace a class-level method with a fixed return value or lambda
+    # for the duration of the block. Used to keep tests away from
+    # external services like the vector embedding web service.
+    def with_stubbed_method(klass, method_name, value_or_lambda)
+      original = klass.method(method_name)
+      klass.define_singleton_method(method_name) do |*args, **kwargs|
+        if value_or_lambda.respond_to?(:call)
+          value_or_lambda.call(*args, **kwargs)
+        else
+          value_or_lambda
+        end
+      end
+      yield
+    ensure
+      klass.define_singleton_method(method_name, original)
+    end
   end
 end