Fix GIT_REV build arg expansion in deploy Makefile

Production imports kept aborting with
ActiveRecord::RecordNotUnique on packages_name_key and - on
environments that never got the constraint - silently created
duplicate packages. Root cause: overlapping update_from_deb_repos
runs (scheduled + manual) both do find-by-name then insert; whichever
wins the race aborts the whole run, whichever loses gets a
duplicate if the constraint was missing.

Three things so the mechanism stops being the problem:

- An advisory lock serializes update_from_deb_repos runs. A second
  run that finds the lock held simply skips with a warning instead
  of racing the first one. The lock is session-scoped, so it is
  released automatically when the process exits.
- A per-package rescue for ActiveRecord::RecordNotUnique. If a
  create still races (e.g. a package being created by a web upload),
  the package is re-fetched and updated instead of aborting the run.
- A real migration for packages_name_key, which previously only
  existed where it had been added by hand. From now on db:migrate
  creates it everywhere, new databases included.

Also verified with a reproduction: a single sequential run of the
deployed importer against a fresh database (both architectures,
~54k packages) creates zero duplicates.

Dockerfile

@@ -5,10 +5,11 @@
 ARG RUBY_VERSION=3.4.7
 ARG NODE_VERSION=22.16.0
 ARG BUN_VERSION=1.3.9
+ARG GIT_REV=unknown
 
 # --- Stage 1: Grab Node.js Binaries ---
 # We use the official image as a source for pre-compiled binaries
-FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source
+FROM docker.io/library/node:${NODE_VERSION} AS node_source
 
 # --- Stage 2: Base (Runtime Environment) ---
 FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base
@@ -36,15 +37,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     apt-get update -qq && \
     apt-get install --no-install-recommends -y \
-    build-essential git libyaml-dev node-gyp pkg-config python-is-python3
+    build-essential git libyaml-dev pkg-config python-is-python3
 
-# "Heist" Node.js and npm from the node_source stage (build-time only)
-COPY --from=node_source /usr/local/bin/node /usr/local/bin/node
-COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules
+# Install Node.js and npm from node_source
+COPY --from=node_source /usr/local/ /usr/local/
 
-# Re-link npm and install Bun (build-time only)
-RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \
-    npm install -g bun@${BUN_VERSION}
+# Install Bun
+RUN npm install -g bun@${BUN_VERSION}
 
 # 1. Install Gems (Separate COPY for maximum caching)
 COPY Gemfile Gemfile.lock ./
@@ -80,8 +79,11 @@ RUN groupadd --system --gid 1000 rails && \
     useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \
     chown -R rails:rails db log tmp public
 
-# Create versioninfo with build timestamp
-RUN date +"%Y-%m-%d %H:%M:%S %Z" > public/versioninfo
+# Redeclare so it is in scope in this final stage
+ARG GIT_REV=unknown
+
+# Create versioninfo with git revision and build timestamp
+RUN printf 'git-rev: %s\nbuilt-at: %s\n' "$GIT_REV" "$(date +'%Y-%m-%d %H:%M:%S %Z')" > public/versioninfo
 
 USER 1000:1000
 

Gemfile

@@ -41,8 +41,7 @@ gem 'mini_magick'
 # Reduces boot times through caching; required in config/boot.rb
 gem 'bootsnap', require: false
 
-# Structured logging for production
-gem 'lograge'
+
 
 # Deploy this application anywhere as a Docker container [https://kamal-deploy.org]
 gem 'kamal', require: false
@@ -109,6 +108,8 @@ group :test do
   # Use system testing [https://guides.rubyonrails.org/testing.html#system-testing]
   gem 'capybara'
   gem 'selenium-webdriver'
+  # Provides the assigns() test helper extracted from modern Rails
+  gem 'rails-controller-testing'
 end
 
 # TODO… https://github.com/galetahub/simple-captcha
@@ -124,6 +125,10 @@ gem 'will_paginate-foundation'
 # Full-text search in PostgreSQL
 gem 'pg_search'
 
+# Nearest neighbor search for PostgreSQL using the pgvector extension
+# https://github.com/ankane/neighbor
+gem 'neighbor'
+
 # Use SLIM as our templating language
 gem 'slim-rails'
 

Gemfile.lock

@@ -224,11 +224,6 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     logger (1.7.0)
-    lograge (0.14.0)
-      actionpack (>= 4)
-      activesupport (>= 4)
-      railties (>= 4)
-      request_store (~> 1.0)
     loofah (2.25.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -255,6 +250,8 @@ GEM
       minitest (>= 5.0, < 7)
       ruby-progressbar
     msgpack (1.8.0)
+    neighbor (1.2.0)
+      activerecord (>= 7.2)
     nenv (0.3.0)
     net-http (0.9.1)
       uri (>= 0.11.1)
@@ -384,6 +381,10 @@ GEM
       activesupport (= 8.1.3)
       bundler (>= 1.15.0)
       railties (= 8.1.3)
+    rails-controller-testing (1.0.5)
+      actionpack (>= 5.0.1.rc1)
+      actionview (>= 5.0.1.rc1)
+      activesupport (>= 5.0.1.rc1)
     rails-dom-testing (2.3.0)
       activesupport (>= 5.0.0)
       minitest
@@ -427,8 +428,6 @@ GEM
     regexp_parser (2.12.0)
     reline (0.6.3)
       io-console (~> 0.5)
-    request_store (1.7.0)
-      rack (>= 1.4)
     responders (3.2.0)
       actionpack (>= 7.0)
       railties (>= 7.0)
@@ -585,10 +584,10 @@ DEPENDENCIES
   jbuilder
   kamal
   listen (~> 3.5)
-  lograge
   mini_magick
   minitest-rails
   minitest-reporters
+  neighbor
   omniauth
   omniauth-rails_csrf_protection
   omniauth_openid_connect
@@ -597,6 +596,7 @@ DEPENDENCIES
   propshaft
   puma (>= 5.0)
   rails (~> 8.1.0)
+  rails-controller-testing
   rails-erd
   rails-healthcheck
   rails_icons (~> 1.3)

Makefile

@@ -2,9 +2,9 @@
 
 -include .env.production .env.development
 
-REGISTRY := registry.coolify1.workaround.org
+REGISTRY := git.workaround.org/chaas
 
-stage: TAG=stage
+stage: TAG=latest
 stage: build tag push
 	curl -s -H "Authorization: Bearer $(COOLIFY_STAGE_DEPLOY_TOKEN)" \
 	  -X POST "$(COOLIFY_STAGE_DEPLOY_WEBHOOK)"
@@ -15,10 +15,11 @@ prod: build tag push
 	  -X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)"
 
 build:
-	podman build -t debshots-$(TAG) .
+	docker build --build-arg GIT_REV=$(shell git rev-parse --short HEAD) -t debshots-$(TAG) .
 
 tag:
-	podman tag debshots-$(TAG) $(REGISTRY)/debshots-web:$(TAG)
+	docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG)
 
 push:
-	podman push $(REGISTRY)/debshots-web:$(TAG)
+	docker push $(REGISTRY)/debshots:$(TAG)
+

README.md

@@ -15,6 +15,42 @@ https://salsa.debian.org/debian/debshots/-/issues
 
 Check out the doc/README.Development.md
 
+## Package search
+
+The package search combines several strategies, tried in this order:
+
+1. **Exact package name**: Searching for `vim` shows exactly this
+   package without calling any external service.
+2. **Name prefix matches**: Packages whose name starts with the query
+   rank first, e.g. searching for `sqlite` promotes `sqlite3`,
+   `sqlitebrowser` and friends above all other results.
+3. **Compound word splitting**: Packages whose name contains *all*
+   query tokens match too, so `sqlite browser` finds `sqlitebrowser`.
+4. **Semantic search**: The query is embedded via the vector service
+   (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among
+   the package description embeddings are returned, using the pgvector
+   extension with an HNSW cosine index. Embeddings are computed from
+   the package description and long description and can be (re-)built
+   with `bin/rails debshots:compute_vectors`.
+5. **Full-text fallback**: If the vector service cannot be reached,
+   classic PostgreSQL full-text search takes over.
+
+Queries that share no vocabulary with the package data (gibberish,
+keyboard mash) return an honest empty result instead of random
+semantic matches.
+
+**Why is this package in my results?** Each result knows which search
+strategy produced it. The controller exposes this as
+`@search_sources` (a hash mapping package ids to `:exact`,
+`:name_prefix`, `:name_contains`, `:semantic` or `:fulltext`). For
+debugging you can render a small badge on every result card by adding
+`show_sources=1` to the URL, e.g.
+`https://screenshots.debian.net/packages?search=sqlite&show_sources=1`.
+
+Package pages also show semantically **related packages**, computed as
+a nearest neighbor query against the package's own description
+embedding - no external service call involved.
+
 ## Deployment
 
 Read the doc/README.Installation.md

app/controllers/packages_controller.rb

@@ -30,6 +30,9 @@ class PackagesController < ApplicationController
       @screenshots = @package.screenshots.accessible_by(current_ability, :view).paginate(
         page: @page, per_page: 6
       )
+      # Semantic recommendations based on the package's own description
+      # embedding - a plain nearest neighbor query, no service call.
+      @related_packages = @package.related_packages
     end
   end
 
@@ -359,10 +362,15 @@ class PackagesController < ApplicationController
   def query_packages
     packages = Package # .order(visits: :desc)
 
+    # Maps package id to the search strategy that put it into the
+    # result list: :exact, :name_prefix, :name_contains, :semantic or
+    # :fulltext. Empty when browsing without a search term.
+    @search_sources = {}
+
     # text search
     if params[:search].present?
       logger.debug "Searching for #{params[:search]}"
-      packages = packages.general_search(params[:search])
+      packages = search_packages(params[:search])
     end
 
     case params[:show]
@@ -378,6 +386,56 @@ class PackagesController < ApplicationController
     packages
   end
 
+  # Text search for packages: an exact package name match wins without
+  # touching the vector service. Otherwise packages whose name starts
+  # with the query (or contains all query tokens, catching compound
+  # words like "sqlite browser" -> "sqlitebrowser") are promoted above
+  # the semantic (vector) nearest neighbor results. Classic PostgreSQL
+  # full-text search kicks in when the vector service cannot be reached.
+  MAX_PROMOTED_NAME_MATCHES = 8
+  MAX_SEMANTIC_RESULTS = 100
+  private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS
+
+  def search_packages(query)
+    exact_match = Package.find_by(name: query)
+    if exact_match
+      @search_sources[exact_match.id] = :exact
+      return Package.where(id: exact_match.id)
+    end
+
+    prefix_matches = Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES).to_a
+    token_matches = Package.name_contains_all(query.to_s.split(/\s+/)).limit(
+      MAX_PROMOTED_NAME_MATCHES
+    ).to_a
+    prefix_matches.each { |package| @search_sources[package.id] ||= :name_prefix }
+    token_matches.each { |package| @search_sources[package.id] ||= :name_contains }
+
+    semantic = []
+    if Package.lexical_overlap?(query)
+      begin
+        semantic = Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a
+        semantic.each { |package| @search_sources[package.id] ||= :semantic }
+      rescue Vectorizer::Error => e
+        logger.error "Semantic search failed: #{e.message}"
+        flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.'
+        semantic = Package.general_search(query).limit(100).to_a
+        semantic.each { |package| @search_sources[package.id] ||= :fulltext }
+      end
+    else
+      logger.debug "Query '#{query}' has no lexical overlap with package data"
+    end
+
+    # Deduplicate by NAME, not id: if the database ever contains
+    # duplicate rows (see the packages_name_key incident) users should
+    # not see the same package twice.
+    results = (prefix_matches + token_matches + semantic).uniq(&:name)
+
+    # Queries that share no vocabulary with our package data can produce
+    # no meaningful results. Skip the vector service round-trip and show
+    # an honest empty result instead of random packages.
+    results.empty? ? Package.none : results
+  end
+
   # Store a random identifier and the client's IP address in the session
   # for later identification.
   # def create_user_token

app/helpers/packages_helper.rb

@@ -1,4 +1,22 @@
 module PackagesHelper
+  # Search result origin badges are only shown when explicitly
+  # requested via ?show_sources=1
+  def search_sources_visible?
+    params[:show_sources].present?
+  end
+
+  # Human readable label for the search strategy that put a package
+  # into the search results (see PackagesController#search_packages)
+  def search_source_label(source)
+    {
+      exact: 'exact match',
+      name_prefix: 'name match',
+      name_contains: 'name match',
+      semantic: 'similar',
+      fulltext: 'full-text'
+    }[source]
+  end
+
   # Return the description and/or the version of the package
   def screenshot_caption(screenshot)
     str = []

app/models/package.rb

@@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open()
 require 'json'
 require 'deb_importer'
 
+# Search and embedding logic keeps accumulating here; the class is
+# cohesive enough that splitting it is not worth the indirection.
+# rubocop:disable Metrics/ClassLength
 class Package < ApplicationRecord
   # PostgreSQL-based full-text search:
   # https://github.com/Casecommons/pg_search
@@ -26,6 +29,11 @@ class Package < ApplicationRecord
            inverse_of: :package,
            dependent: :destroy
 
+  # Nearest neighbor search using pgvector. The "embedding" column stores
+  # a vector computed from description + long_description by an external
+  # web service (see Vectorizer and lib/tasks/vectorize_packages.rake).
+  has_neighbors :embedding
+
   # default_scope {
   # order('name ASC')
   # }
@@ -40,6 +48,103 @@ class Package < ApplicationRecord
     long_description.split(/\n\.\n/).first if long_description
   end
 
+  # The text that the embedding vector is computed from: description
+  # and long description combined.
+  def embedding_text
+    [description, long_description].compact.join("\n")
+  end
+
+  # Compute and store the embedding vector for this package by asking
+  # the external vector service. Raises Vectorizer::Error on failure.
+  # Returns true if a vector was saved.
+  def update_embedding!
+    text = embedding_text
+    return false if text.blank?
+
+    update_column(:embedding, Vectorizer.embed(text))
+  end
+
+  # Return a relation of packages whose stored embeddings are closest to
+  # the given query vector (an array of floats). Records get a
+  # "neighbor_distance" attribute added (cosine distance, lower is better).
+  # The returned relation can still be chained (e.g. filtered or paginated).
+  #
+  # Only returns packages that already have an embedding computed.
+  def self.nearest_to_vector(vector, limit: nil)
+    neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine')
+    limit ? neighbors.limit(limit) : neighbors
+  end
+
+  # Turn a free-text search string into a vector via the external web
+  # service and find the closest packages.
+  def self.nearest_to_text(text, limit: nil)
+    return none if text.blank?
+
+    nearest_to_vector(Vectorizer.embed(text), limit: limit)
+  end
+
+  # Packages whose name starts with the given string - used to promote
+  # obvious matches like "sqlite3" for a "sqlite" search.
+  def self.name_starts_with(text)
+    return none if text.blank?
+
+    where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name)
+  end
+
+  # Packages whose name contains all given tokens as substrings. Catches
+  # compound words split by the user, e.g. "sqlite browser" finding the
+  # "sqlitebrowser" package.
+  def self.name_contains_all(tokens)
+    tokens = tokens.select { |token| token.length >= 2 }
+    return none if tokens.empty?
+
+    scope = self
+    tokens.each do |token|
+      scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\')
+    end
+    scope.order(:name)
+  end
+
+  def self.escape_for_like(text)
+    # Escape LIKE wildcards so that package name characters like "+" or "."
+    # cannot break the pattern
+    text.gsub(/[\\%_]/) { |char| "\\#{char}" }
+  end
+
+  # Packages semantically similar to this one, based on the stored
+  # description embedding. Returns an empty relation for packages that
+  # have no embedding (yet).
+  def related_packages(limit: 6)
+    return Package.none unless embedding
+
+    Package.nearest_to_vector(embedding, limit: limit + 1)
+           .where.not(id: id).limit(limit)
+  end
+
+  # Check whether the given text shares at least one lexeme with any
+  # package name/description in the database. Queries without such an
+  # overlap (gibberish, keyboard mash, unsupported languages) can
+  # neither match full-text nor produce meaningful semantic results -
+  # the embedding model maps unknown tokens close to the corpus
+  # centroid, making them look like good matches.
+  def self.lexical_overlap?(text)
+    tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) }
+    return false if tokens.empty?
+
+    # Pin the same text search dictionary that the packages_fts index
+    # uses - the server-wide default can be anything.
+    tsquery = tokens.map do |token|
+      sanitize_sql(["plainto_tsquery('english', ?)", token])
+    end.join(' || ')
+    # The expression must match the packages_fts GIN index definition
+    # so that Postgres can use the index.
+    where(
+      "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \
+      "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \
+      "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})"
+    ).exists?
+  end
+
   # Return a query of all packages that have screenshots
   def self.with_screenshots
     # Query for all packages who's ID appears in a screenshot's "package_id" field
@@ -113,3 +218,4 @@ class Package < ApplicationRecord
     end
   end
 end
+# rubocop:enable Metrics/ClassLength

app/views/packages/browse.slim

@@ -14,7 +14,7 @@
     - if @packages.any?
       - if @view_style==:grid
         .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium"
-          - @packages.all.each do |pkg|
+          - @packages.each do |pkg|
             .cell.pkgcard data-equalizer-watch=true
               a.black href=package_path(name: pkg.name)
                 .image
@@ -24,6 +24,9 @@
                   = pkg.name
                 .text
                   = pkg.description
+                - if search_sources_visible? && @search_sources[pkg.id]
+                  .text.search-source
+                    span.label.secondary = search_source_label(@search_sources[pkg.id])
       - elsif @view_style==:list
         - @packages.to_a.each do |pkg|
           .grid-x.grid-margin-x.listview
@@ -37,6 +40,9 @@
                 a href=package_path(name: pkg.name)
                   =pkg.name
               p =pkg.description
+              - if search_sources_visible? && @search_sources[pkg.id]
+                p
+                  span.label.secondary = search_source_label(@search_sources[pkg.id])
               .listview.longdescription
                 = pkg.long_description_first_paragraph
       

app/views/packages/details.slim

@@ -70,6 +70,19 @@
       / #metadata is the anchor for this sticky container.
       = render(partial: 'details_rightbox', locals: {pkg: @package})
 
+  / Semantically similar packages, based on the package's description embedding
+  - if @related_packages.any?
+    #related
+      h3 Related packages
+      .grid-x.grid-margin-x.medium-up-2.large-up-3 data-equalizer=true
+        - @related_packages.each do |pkg|
+          .cell.pkgcard data-equalizer-watch=true
+            a.black href=package_path(name: pkg.name)
+              .text.pkgname
+                = pkg.name
+              .text.small
+                = pkg.description
+
 javascript:
   // Where to send POST requests for uploads
   let ajax_upload_url = '#{{upload_receive_json_path}}';

config/environments/production.rb

@@ -37,15 +37,6 @@ Rails.application.configure do
   config.log_tags = [:request_id]
   config.logger   = ActiveSupport::TaggedLogging.logger(STDOUT)
 
-  # Lograge - structured request logging
-  config.lograge.enabled = true
-  config.lograge.custom_options = lambda do |event|
-    {
-      request_id: event.payload[:request_id],
-      user_id: event.payload[:user_id]
-    }
-  end
-
   # Change to "debug" to log everything (including potentially personally-identifiable information!).
   config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info')
 

config/initializers/will_paginate.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Enable will_paginate's Array#paginate so that merged search results
+# (name prefix matches + semantic results) can be paginated like
+# ActiveRecord relations.
+require 'will_paginate/array'

db/migrate/20260821203833_install_neighbor_vector.rb

@@ -0,0 +1,5 @@
+class InstallNeighborVector < ActiveRecord::Migration[8.1]
+  def change
+    enable_extension "vector"
+  end
+end

db/migrate/20260821210000_add_embedding_to_packages.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Adds a vector column to store text embeddings of the package
+# description and long description. The vectors are computed by an
+# external web service and used for nearest neighbor search.
+#
+# The pgvector extension is enabled by the InstallNeighborVector migration.
+class AddEmbeddingToPackages < ActiveRecord::Migration[8.1]
+  def change
+    add_column :packages, :embedding, :vector, limit: 384
+
+    # HNSW index for fast approximate nearest neighbor searches
+    add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops
+  end
+end

db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Packages must be unique by name - the same package is listed once per
+# architecture/component in the Debian archive and the importer looks it up
+# by name before inserting, so anything that lets two rows for one name appear
+# (e.g. concurrent import runs) breaks the site. Production had accidentally
+# relied on a constraint that was only ever added ad-hoc on some databases
+# and never tracked as a migration; this makes db:migrate the single source
+# of truth.
+class AddUniqueNameConstraintToPackages < ActiveRecord::Migration[8.1]
+  def up
+    add_unique_constraint :packages, :name, name: :packages_name_key unless constraint_exists?
+  end
+
+  def down
+    remove_unique_constraint :packages, :name, name: :packages_name_key if constraint_exists?
+  end
+
+  private
+
+  def constraint_exists?
+    select_value(<<~SQL.squish)
+      SELECT 1
+      FROM pg_constraint
+      WHERE conrelid = 'packages'::regclass AND conname = 'packages_name_key' AND contype = 'u'
+    SQL
+  end
+end

db/schema.rb

@@ -10,133 +10,131 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do
-  # These are extensions that must be enabled to support this database
-  enable_extension 'pg_catalog.plpgsql'
+ActiveRecord::Schema[8.1].define(version: 2026_09_04_120000) do
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "pg_catalog.plpgsql"
+  enable_extension "vector"
 
-  create_table 'action_mailbox_inbound_emails', force: :cascade do |t|
-    t.integer 'status', default: 0, null: false
-    t.string 'message_id', null: false
-    t.string 'message_checksum', null: false
-    t.datetime 'created_at', null: false
-    t.datetime 'updated_at', null: false
-    t.index ['message_id', 'message_checksum'], 
-name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true
+  create_table "action_mailbox_inbound_emails", force: :cascade do |t|
+    t.datetime "created_at", null: false
+    t.string "message_checksum", null: false
+    t.string "message_id", null: false
+    t.integer "status", default: 0, null: false
+    t.datetime "updated_at", null: false
+    t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true
   end
 
-  create_table 'action_text_rich_texts', force: :cascade do |t|
-    t.string 'name', null: false
-    t.text 'body'
-    t.string 'record_type', null: false
-    t.bigint 'record_id', null: false
-    t.datetime 'created_at', null: false
-    t.datetime 'updated_at', null: false
-    t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', 
-unique: true
+  create_table "action_text_rich_texts", force: :cascade do |t|
+    t.text "body"
+    t.datetime "created_at", null: false
+    t.string "name", null: false
+    t.bigint "record_id", null: false
+    t.string "record_type", null: false
+    t.datetime "updated_at", null: false
+    t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true
   end
 
-  create_table 'active_storage_attachments', force: :cascade do |t|
-    t.string 'name', null: false
-    t.string 'record_type', null: false
-    t.bigint 'record_id', null: false
-    t.bigint 'blob_id', null: false
-    t.datetime 'created_at', precision: nil, null: false
-    t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id'
-    t.index ['record_type', 'record_id', 'name', 'blob_id'], 
-name: 'index_active_storage_attachments_uniqueness', unique: true
+  create_table "active_storage_attachments", force: :cascade do |t|
+    t.bigint "blob_id", null: false
+    t.datetime "created_at", precision: nil, null: false
+    t.string "name", null: false
+    t.bigint "record_id", null: false
+    t.string "record_type", null: false
+    t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id"
+    t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true
   end
 
-  create_table 'active_storage_blobs', force: :cascade do |t|
-    t.string 'key', null: false
-    t.string 'filename', null: false
-    t.string 'content_type'
-    t.text 'metadata'
-    t.bigint 'byte_size', null: false
-    t.string 'checksum'
-    t.datetime 'created_at', precision: nil, null: false
-    t.string 'service_name', null: false
-    t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true
+  create_table "active_storage_blobs", force: :cascade do |t|
+    t.bigint "byte_size", null: false
+    t.string "checksum"
+    t.string "content_type"
+    t.datetime "created_at", precision: nil, null: false
+    t.string "filename", null: false
+    t.string "key", null: false
+    t.text "metadata"
+    t.string "service_name", null: false
+    t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true
   end
 
-  create_table 'active_storage_variant_records', force: :cascade do |t|
-    t.bigint 'blob_id', null: false
-    t.string 'variation_digest', null: false
-    t.index ['blob_id', 'variation_digest'], 
-name: 'index_active_storage_variant_records_uniqueness', unique: true
+  create_table "active_storage_variant_records", force: :cascade do |t|
+    t.bigint "blob_id", null: false
+    t.string "variation_digest", null: false
+    t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true
   end
 
-  create_table 'logs', id: :serial, force: :cascade do |t|
-    t.string 'message'
-    t.string 'level'
-    t.string 'section'
-    t.inet 'ip_address'
-    t.datetime 'created_at', precision: nil, null: false
-    t.datetime 'updated_at', precision: nil, null: false
-    t.integer 'user_id'
-    t.integer 'package_id'
-    t.integer 'screenshot_id'
+  create_table "logs", id: :serial, force: :cascade do |t|
+    t.datetime "created_at", precision: nil, null: false
+    t.inet "ip_address"
+    t.string "level"
+    t.string "message"
+    t.integer "package_id"
+    t.integer "screenshot_id"
+    t.string "section"
+    t.datetime "updated_at", precision: nil, null: false
+    t.integer "user_id"
   end
 
-  create_table 'packages', id: :serial, force: :cascade do |t|
-    t.string 'name', null: false
-    t.string 'description', limit: 80
-    t.string 'section', limit: 50
-    t.string 'maintainer', limit: 100
-    t.string 'maintainer_email', limit: 100
-    t.string 'homepage', limit: 400
-    t.string 'version', limit: 200
-    t.text 'long_description'
-    t.string 'origin', limit: 80
-    t.datetime 'created_at', precision: nil
-    t.datetime 'updated_at', precision: nil
-    t.integer 'visits', default: 0
-    t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", 
-name: 'packages_fts', using: :gin
-    t.unique_constraint ['name'], name: 'packages_name_key'
+  create_table "packages", id: :serial, force: :cascade do |t|
+    t.datetime "created_at", precision: nil
+    t.string "description", limit: 80
+    t.vector "embedding", limit: 384
+    t.string "homepage", limit: 400
+    t.text "long_description"
+    t.string "maintainer", limit: 100
+    t.string "maintainer_email", limit: 100
+    t.string "name", null: false
+    t.string "origin", limit: 80
+    t.string "section", limit: 50
+    t.datetime "updated_at", precision: nil
+    t.string "version", limit: 200
+    t.integer "visits", default: 0
+    t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin
+    t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw
+    t.unique_constraint ["name"], name: "packages_name_key"
   end
 
-  create_table 'screenshots', id: :serial, force: :cascade do |t|
-    t.integer 'package_id'
-    t.string 'version', limit: 50
-    t.datetime 'created_at', precision: nil
-    t.string 'uploaderhash', limit: 72
-    t.boolean 'approved', default: false, null: false
-    t.text 'description'
-    t.datetime 'updated_at', precision: nil
-    t.string 'image_fingerprint'
-    t.integer 'user_id', default: 0
-    t.text 'simage_data'
-    t.inet 'uploaderip'
-    t.boolean 'hidden', default: false
-    t.index ['id', 'approved'], name: 'id_approved'
-    t.index ['id', 'uploaderhash'], name: 'id_uploaderhash'
+  create_table "screenshots", id: :serial, force: :cascade do |t|
+    t.boolean "approved", default: false, null: false
+    t.datetime "created_at", precision: nil
+    t.text "description"
+    t.boolean "hidden", default: false
+    t.string "image_fingerprint"
+    t.integer "package_id"
+    t.text "simage_data"
+    t.datetime "updated_at", precision: nil
+    t.string "uploaderhash", limit: 72
+    t.inet "uploaderip"
+    t.integer "user_id", default: 0
+    t.string "version", limit: 50
+    t.index ["id", "approved"], name: "id_approved"
+    t.index ["id", "uploaderhash"], name: "id_uploaderhash"
   end
 
-  create_table 'users', id: :serial, force: :cascade do |t|
-    t.text 'name'
-    t.datetime 'created_at', precision: nil, null: false
-    t.datetime 'updated_at', precision: nil, null: false
-    t.string 'email', default: '', null: false
-    t.string 'encrypted_password', default: '', null: false
-    t.integer 'sign_in_count', default: 0, null: false
-    t.datetime 'current_sign_in_at', precision: nil
-    t.datetime 'last_sign_in_at', precision: nil
-    t.inet 'current_sign_in_ip'
-    t.inet 'last_sign_in_ip'
-    t.integer 'failed_attempts', default: 0, null: false
-    t.string 'unlock_token'
-    t.datetime 'locked_at', precision: nil
-    t.string 'provider'
-    t.string 'uid'
-    t.boolean 'admin_role', default: false
-    t.boolean 'moderator_role', default: false
-    t.boolean 'pseudo', default: false
-    t.integer 'approved_screenshots', default: 0
-    t.integer 'rejected_screenshots', default: 0
-    t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true
+  create_table "users", id: :serial, force: :cascade do |t|
+    t.boolean "admin_role", default: false
+    t.integer "approved_screenshots", default: 0
+    t.datetime "created_at", precision: nil, null: false
+    t.datetime "current_sign_in_at", precision: nil
+    t.inet "current_sign_in_ip"
+    t.string "email", default: "", null: false
+    t.string "encrypted_password", default: "", null: false
+    t.integer "failed_attempts", default: 0, null: false
+    t.datetime "last_sign_in_at", precision: nil
+    t.inet "last_sign_in_ip"
+    t.datetime "locked_at", precision: nil
+    t.boolean "moderator_role", default: false
+    t.text "name"
+    t.string "provider"
+    t.boolean "pseudo", default: false
+    t.integer "rejected_screenshots", default: 0
+    t.integer "sign_in_count", default: 0, null: false
+    t.string "uid"
+    t.string "unlock_token"
+    t.datetime "updated_at", precision: nil, null: false
+    t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true
   end
 
-  add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id'
-  add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id'
-  add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey'
+  add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
+  add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
+  add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey"
 end

lib/tasks/import_debian.rake

@@ -29,11 +29,47 @@ BLACKLIST_SECTION_PATTERN = [
   %r{/?oldlibs$},
   %r{/?libdevel$},
   %r{/?cli-mono$},
+  # Libraries never make useful screenshots - and unlike a "lib*" name
+  # pattern this does not hit applications like libreoffice (editors)
+  # or librecad (graphics)
+  %r{/?libs$},
+]
+
+# List of regular expressions. If the package's short description
+# matches any of these then the package will not be imported.
+# Transitional/dummy stub packages exist only to ease upgrades to a
+# renamed or split package and never get a useful screenshot. Debian
+# has no dedicated metadata field for this - the Developer's Reference
+# (6.9.7) documents these exact phrases as the convention package
+# maintainers use, and as what deborphan --guess-dummy looks for.
+# Deliberately not matching a bare "dummy": too many real packages use
+# that word for actual functionality (e.g. xserver-xorg-video-dummy,
+# fence-agents-dummy, "double dummy solver" bridge card libraries).
+BLACKLIST_DESCRIPTION_PATTERN = [
+  /transitional/i,
+  /dummy package/i,
+  /dependency package/i,
 ]
 
 # Whether to delete a blacklisted package from the database
 REMOVE_BLACKLISTED_PACKAGE = true
 
+# Postgres advisory lock used to serialize update_from_deb_repos runs.
+# Without this, a scheduled and a manual run (or two scheduled runs) can
+# overlap: both do find_by+insert for the same name, and the second one
+# either violates packages_name_key (aborting the whole import) or - on
+# instances that never had that constraint - silently creates a duplicate.
+IMPORT_ADVISORY_LOCK_KEY = 0x0DE65507
+
+# Whether to delete packages that were not found in any configured
+# source during a full import run (e.g. packages Debian has dropped
+# entirely, or ones from a suite/architecture no longer configured).
+# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking
+# "orphaned" can also mean a mirror had a transient fetch problem, so
+# update_from_deb_repos refuses to act on it unless every configured
+# component/architecture was actually fetched successfully this run.
+REMOVE_ORPHANED_PACKAGES = false
+
 namespace :debshots do
   desc "Import/update package database from configured DEB repositories"
 
@@ -43,11 +79,23 @@ namespace :debshots do
     stats_updated = 0
     stats_removed = 0
 
+    # Every package name encountered in any configured source this run,
+    # used to find orphans afterwards. A Hash is used as a cheap set.
+    seen_package_names = {}
+    # Set to true if any component/architecture could not be fetched,
+    # so we know not to trust seen_package_names for orphan removal.
+    sources_incomplete = false
+
     repositories = Rails.configuration.package_sources
     Rails.logger = Logger.new(STDOUT)
     Rails.logger.level = Logger::INFO
     #Rails.logger.level = Logger::DEBUG
 
+    unless import_lock_acquired?
+      Rails.logger.warn 'Another import run already in progress (advisory lock). Skipping.'
+      exit 0
+    end
+
     Rails.logger.info "Importing Debian package information"
 
     repositories.each do |repository|
@@ -74,6 +122,7 @@ namespace :debshots do
           packages = release.packages(component, architecture)
           unless packages
             Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
+            sources_incomplete = true
             next
           end
 
@@ -82,8 +131,11 @@ namespace :debshots do
           packages.each do |package|
             Rails.logger.info "> Package: #{package[:Package]}"
             #Rails.logger.debug "Fetching package informaton from the database"
+            seen_package_names[package[:Package]] = true
 
-            if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section]
+            if package_name_blacklisted?(package[:Package]) or
+               package_section_blacklisted?(package[:Section]) or
+               package_description_blacklisted?(package[:Description])
               # Should the package get removed from the database?
               if REMOVE_BLACKLISTED_PACKAGE
                 db_package = Package.find_by name: package[:Package]
@@ -97,24 +149,37 @@ namespace :debshots do
               next
             end # if blacklisted
 
-            db_package = Package.find_by name: package[:Package]
-            if db_package  # package exists in the database
-              # check if the parsed package data has a newer version
-              current_version = DebImporter::Version.new(db_package.version)
-              new_version = DebImporter::Version.new(package[:Version])
+            begin
+              db_package = Package.find_by name: package[:Package]
+              if db_package  # package exists in the database
+                # check if the parsed package data has a newer version
+                current_version = DebImporter::Version.new(db_package.version)
+                new_version = DebImporter::Version.new(package[:Version])
 
-              Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
-              if new_version > current_version
-                # update the package information from new data
+                Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
+                if new_version > current_version
+                  # update the package information from new data
+                  update_data(package, db_package)
+                  Rails.logger.info "Updating package information"
+                  stats_updated += 1
+                end
+              else # new package - create it in the database
+                Rails.logger.debug "No such package in our database. Creating one."
+                db_package = Package.new
+                update_data(package, db_package)
+                stats_added += 1
+              end
+            rescue ActiveRecord::RecordNotUnique
+              # Lost a create-race: another process inserted this package name
+              # between our find and our insert (a web upload, or an import on
+              # an instance without the advisory lock). Treat it as found and
+              # update instead of aborting the whole run.
+              Rails.logger.warn "Package '#{package[:Package]}' was created " \
+                                 'concurrently. Updating it instead.'
+              if (db_package = Package.find_by(name: package[:Package]))
                 update_data(package, db_package)
-                Rails.logger.info "Updating package information"
                 stats_updated += 1
               end
-            else # new package - create it in the database
-              Rails.logger.debug "No such package in our database. Creating one."
-              db_package = Package.new
-              update_data(package, db_package)
-              stats_added += 1
             end
 
             # unless db_package
@@ -138,6 +203,9 @@ namespace :debshots do
       Rails.logger.info "--------------------------------------------"
     end # repositories.each
 
+    stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete)
+    Rails.logger.info "#{stats_removed} packages removed in total"
+
   end # task
 
   desc 'Update long description from i18n file'
@@ -276,3 +344,80 @@ def package_section_blacklisted?(section)
   end
   return false
 end
+
+# Check if a package's short description marks it as a transitional or
+# dummy stub package (e.g. "transitional package", "transitional dummy
+# package for foo"). These never get a useful screenshot.
+def package_description_blacklisted?(description)
+  BLACKLIST_DESCRIPTION_PATTERN.each do |pattern|
+    if description.to_s =~ pattern
+      Rails.logger.debug "  > Blacklisted by description ('#{description}' matches '#{pattern}')"
+      return true
+    end
+  end
+  false
+end
+
+# Remove packages that were not encountered in any configured source
+# during this import run. Refuses to act if any component/architecture
+# failed to fetch (sources_incomplete) or if seen_names came back empty
+# (both would otherwise risk wiping out packages that are still very
+# much alive in Debian, just missed because of a network hiccup).
+# Returns the number of packages actually removed.
+def orphan_removal_blocked?(seen_names, sources_incomplete)
+  if sources_incomplete
+    Rails.logger.warn '  > Skipping: not all configured components/architectures could be ' \
+                       'fetched this run. Re-run when all sources are reachable.'
+    return true
+  end
+
+  if seen_names.empty?
+    Rails.logger.warn '  > Skipping: no packages were seen this run at all - refusing to ' \
+                       'treat that as "everything is orphaned".'
+    return true
+  end
+
+  false
+end
+
+def find_orphaned_package_ids(seen_names)
+  Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids|
+    ids << id unless seen_names.key?(name)
+  end
+end
+
+def destroy_orphaned_packages(ids)
+  removed = 0
+  Package.where(id: ids).find_each do |package|
+    Rails.logger.info "  > Removing orphaned package '#{package.name}' from database"
+    package.destroy
+    removed += 1
+  end
+  removed
+end
+
+def remove_orphaned_packages(seen_names, sources_incomplete)
+  Rails.logger.info 'Checking for packages no longer found in any configured source'
+  return 0 if orphan_removal_blocked?(seen_names, sources_incomplete)
+
+  orphan_ids = find_orphaned_package_ids(seen_names)
+
+  unless REMOVE_ORPHANED_PACKAGES
+    Rails.logger.info "  > #{orphan_ids.size} packages are orphaned (dry run - set " \
+                       'REMOVE_ORPHANED_PACKAGES to actually remove them)'
+    return 0
+  end
+
+  destroy_orphaned_packages(orphan_ids)
+end
+
+# Try to take the advisory lock guarding update_from_deb_repos. The lock lives
+# on this database connection/session, so it is released automatically when the
+# process exits - no explicit unlock needed. Returns true if this run is the
+# one that may proceed.
+def import_lock_acquired?
+  result = ActiveRecord::Base.connection.execute(
+    "SELECT pg_try_advisory_lock(#{IMPORT_ADVISORY_LOCK_KEY})"
+  )
+  result.first['pg_try_advisory_lock'] == true
+end

lib/tasks/vectorize_packages.rake

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+# Long rake task bodies are normal; the block-length cop is meant for
+# application code.
+# rubocop:disable Metrics/BlockLength
+namespace :debshots do
+  desc 'Compute embedding vectors for all packages based on their description ' \
+       'and long description (missing ones only unless FORCE=1)'
+  task compute_vectors: :environment do
+    logger = Logger.new($stdout)
+    logger.level = Logger::INFO
+
+    force = ENV['FORCE'].present?
+    # Packages without any description text cannot be embedded at all -
+    # exclude them so they do not fail on every single run.
+    base_scope = Package.where("COALESCE(description, '') <> '' OR " \
+                               "COALESCE(long_description, '') <> ''")
+    scope = force ? base_scope : base_scope.where(embedding: nil)
+    total = scope.count
+
+    if total.zero?
+      logger.info 'No packages need vector computation.'
+      next
+    end
+
+    logger.info "Computing vectors for #{total} packages " \
+                "(#{force ? 'recomputing' : 'missing only'}, " \
+                "service: #{Vectorizer.service_url})"
+
+    done = 0
+    failed = 0
+
+    # Remember the most recent SQL statement so failures can be traced
+    # to the exact query that triggered them.
+    last_sql = nil
+    ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
+      last_sql = args.last[:sql]
+    end
+
+    scope.find_each do |package|
+      begin
+        raise ArgumentError, 'no description text' if package.embedding_text.blank?
+
+        package.update_embedding!
+        done += 1
+      rescue StandardError => e
+        # A single broken package must never stop the whole run - log
+        # enough detail to diagnose it and move on.
+        failed += 1
+        logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}"
+        logger.error "Last SQL: #{last_sql}"
+        logger.error e.backtrace.first(15).join("\n") if e.backtrace
+      end
+
+      processed = done + failed
+      next unless (processed % 100).zero? || processed == total
+
+      percent = (processed.to_f / total * 100).round(1)
+      logger.info "Progress: #{processed}/#{total} (#{percent}%)"
+    end
+
+    logger.info "Done. #{done} vectors computed, #{failed} failures."
+  end
+end
+# rubocop:enable Metrics/BlockLength

lib/vectorizer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+# Client for the external web service that turns text into embedding
+# vectors for nearest neighbor search (pgvector).
+#
+#   Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...]
+#
+# The service URL can be overridden with the VECTOR_SERVICE_URL
+# environment variable.
+class Vectorizer
+  # Raised when the vector service cannot be reached or returns garbage
+  class Error < StandardError; end
+
+  DEFAULT_URL = 'https://vector-search.workaround.org/vector'
+
+  class << self
+    # Compute the embedding vector for the given text.
+    # Returns an array of floats.
+    def embed(text)
+      new.embed(text)
+    end
+
+    def service_url
+      ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL)
+    end
+  end
+
+  def embed(text)
+    response = post_text(text)
+
+    unless response.is_a?(Net::HTTPSuccess)
+      raise Error, "Vector service returned HTTP #{response.code}"
+    end
+
+    JSON.parse(response.body).fetch('vector')
+  rescue JSON::ParserError => e
+    raise Error, "Vector service returned invalid JSON: #{e.message}"
+  rescue KeyError
+    raise Error, 'Vector service response did not contain a "vector" field'
+  end
+
+  private
+
+  def post_text(text)
+    uri = URI.parse(self.class.service_url)
+    request = build_request(uri, text)
+
+    options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 }
+    Net::HTTP.start(uri.host, uri.port, **options) do |http|
+      http.request(request)
+    end
+  rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e
+    raise Error, "Could not reach vector service: #{e.message}"
+  end
+
+  def build_request(uri, text)
+    request = Net::HTTP::Post.new(uri.request_uri)
+    request['Content-Type'] = 'application/json'
+    request.body = { text: text }.to_json
+
+    # Optional Bearer token if the service runs with API_SECRET set
+    secret = ENV['VECTOR_SERVICE_SECRET']
+    request['Authorization'] = "Bearer #{secret}" if secret.present?
+
+    request
+  end
+end

test/controllers/packages_controller_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackagesControllerTest < ActionController::TestCase
@@ -20,6 +22,118 @@ class PackagesControllerTest < ActionController::TestCase
     assert_select 'div', 'A program to browse web sites.'
   end
 
+  test 'exact package name search wins without calling the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'vim', show_sources: '1' }
+      assert_response :success
+      assert_select 'div', 'vim'
+      assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources))
+      assert_select '.search-source .label', 'exact match'
+    end
+  end
+
+  test 'gibberish search shows no results and never calls the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'asdfgh zzzz' }
+      assert_response :success
+      # the grid renders every package in the database when no search
+      # filter applies - so an empty result must not fall back to that
+      assert_select 'div', text: /package-\d+/, count: 0
+      assert_empty assigns(:search_sources)
+    end
+  end
+
+  test 'name prefix matches rank above semantic results' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do
+      get :grid, params: { search: 'package' }
+      assert_response :success
+      # fixtures create package-0..package-99 - prefix hits must render
+      assert_select 'div', 'package-0'
+      # ...and before any semantically found package
+      assert response.body.index('package-0') < response.body.index('vim')
+      sources = assigns(:search_sources)
+      assert_equal :name_prefix, sources[Package.find_by(name: 'package-0').id]
+      assert_equal :semantic, sources[packages(:vim).id]
+    end
+  end
+
+  test 'prefix matches and semantic results are deduplicated' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do
+      get :grid, params: { search: 'fire' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+      # the higher tier wins the attribution
+      assert_equal :name_prefix, assigns(:search_sources)[packages(:firefox).id]
+    end
+  end
+
+  test 'compound words in search match packages with joined names' do
+    # "fire fox" should find "firefox" via name substring matching
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.none }) do
+      get :grid, params: { search: 'fire fox', show_sources: '1' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+      assert_equal({ packages(:firefox).id => :name_contains },
+                   assigns(:search_sources))
+      assert_select '.search-source .label', 'name match'
+    end
+  end
+
+  test 'search source badges stay hidden without show_sources param' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.none }) do
+      get :grid, params: { search: 'fire fox' }
+      assert_response :success
+      assert_select '.search-source', count: 0
+    end
+  end
+
+  test 'details shows semantically related packages' do
+    packages(:vim).update_column(:embedding, Array.new(384, 0.25))
+    packages(:firefox).update_column(:embedding, Array.new(384, 0.75))
+
+    get :details, params: { name: 'vim' }
+    assert_response :success
+    assert_select '#related .pkgname', text: 'firefox'
+  end
+
+  test 'details without embedding renders no related section' do
+    get :details, params: { name: 'vim' }
+    assert_response :success
+    assert_select '#related', count: 0
+  end
+
+  test 'search uses semantic (vector) results' do
+    relation = Package.where(name: packages(:vim).name)
+    with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+      assert_equal({ packages(:vim).id => :semantic }, assigns(:search_sources))
+    end
+  end
+
+  # There is deliberately no fallback for an empty vector search
+  # result: KNN always returns something as long as packages have
+  # embeddings. Full-text search is only used when the vector service
+  # cannot be reached.
+
+  test 'search falls back to full-text search when vector service fails' do
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { raise Vectorizer::Error, 'service down' }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+      assert flash[:error].present?
+      assert_equal({ packages(:vim).id => :fulltext }, assigns(:search_sources))
+    end
+  end
+
   # test 'should get thumbnail for a package and a desired version' do
   #   # get '/thumbnail-with-version/package-5/0.1'
   #   get thumbnail_with_version_url('package-5', '0.1')

test/models/package_test.rb

@@ -1,7 +1,107 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackageTest < ActiveSupport::TestCase
-  # test "the truth" do
-  #   assert true
-  # end
+  test 'embedding_text combines description and long description' do
+    package = packages(:firefox)
+    assert_equal "A web browser\nA program to browse web sites.", package.embedding_text
+  end
+
+  test 'embedding_text works with missing long description' do
+    package = packages(:firefox)
+    package.long_description = nil
+    assert_equal 'A web browser', package.embedding_text
+  end
+
+  test 'update_embedding! stores the vector returned by the vector service' do
+    vector = Array.new(384) { |i| i / 384.0 }
+    with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do
+      package = packages(:vim)
+      assert package.update_embedding!
+      # The vector type stores float32 so allow small rounding differences
+      package.reload.embedding.to_a.each_with_index do |actual, i|
+        assert_in_delta vector[i], actual, 1e-6
+      end
+    end
+  end
+
+  test 'update_embedding! does nothing without description text' do
+    package = packages(:firefox)
+    package.description = nil
+    package.long_description = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { flunk 'should not call the vector service' }) do
+      assert_not package.update_embedding!
+    end
+  end
+
+  test 'nearest_to_text returns none for blank text' do
+    assert_empty Package.nearest_to_text('')
+    assert_empty Package.nearest_to_text(nil)
+  end
+
+  test 'name_starts_with matches name prefixes case-insensitively' do
+    assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name
+    assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name
+    assert_empty Package.name_starts_with('')
+    assert_empty Package.name_starts_with(nil)
+  end
+
+  test 'name_contains_all finds compound words split by spaces' do
+    assert_includes Package.name_contains_all(%w[fire fox]).map(&:name),
+                    packages(:firefox).name
+    # all tokens must be present in the name
+    assert_empty Package.name_contains_all(%w[firefox zzzz])
+    assert_empty Package.name_contains_all([])
+    # single character tokens would flood results - ignore them
+    assert_empty Package.name_contains_all(['f'])
+  end
+
+  test 'lexical_overlap? pins the english dictionary regardless of server config' do
+    # Regression: with default_text_search_config = german (as on some
+    # servers) 'browser' stems to 'brows' which never matches the
+    # english-stemmed index.
+    connection = ActiveRecord::Base.connection
+    original = connection.select_value('SHOW default_text_search_config')
+    connection.execute("SET default_text_search_config = 'german'")
+    assert Package.lexical_overlap?('browser')
+  ensure
+    connection&.execute("SET default_text_search_config = '#{original}'")
+  end
+
+  test 'lexical_overlap? detects real words vs gibberish' do
+    # words present in fixture package descriptions
+    assert Package.lexical_overlap?('browser')
+    assert Package.lexical_overlap?('email client for vim syntax')
+    assert Package.lexical_overlap?('the browser') # stopwords are ignored
+    # gibberish, numbers and empty strings share no vocabulary
+    assert_not Package.lexical_overlap?('asdfgh zzzz')
+    assert_not Package.lexical_overlap?('123 !!!')
+    assert_not Package.lexical_overlap?('the') # stopword-only query
+    assert_not Package.lexical_overlap?('')
+    assert_not Package.lexical_overlap?(nil)
+  end
+
+  test 'nearest_to_text embeds the query and searches' do
+    vector = Array.new(384, 0.25)
+    embed_called_with = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        lambda { |text|
+                          embed_called_with = text
+                          vector
+                        }) do
+      received = []
+      results = with_stubbed_method(Package, :nearest_neighbors,
+                                    lambda { |_field, received_vector, **_opts|
+                                      received << received_vector
+                                      [packages(:vim)]
+                                    }) do
+        Package.nearest_to_text('some search text')
+      end
+      assert_equal [packages(:vim)], results
+      assert_equal 'some search text', embed_called_with
+      assert_equal [vector], received
+    end
+  end
 end

test/test_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ENV['RAILS_ENV'] ||= 'test'
 require File.expand_path('../config/environment', __dir__)
 require 'rails/test_help'
@@ -15,6 +17,23 @@ module ActiveSupport
   class TestCase
     # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
     fixtures :all
+
+    # Replace a class-level method with a fixed return value or lambda
+    # for the duration of the block. Used to keep tests away from
+    # external services like the vector embedding web service.
+    def with_stubbed_method(klass, method_name, value_or_lambda)
+      original = klass.method(method_name)
+      klass.define_singleton_method(method_name) do |*args, **kwargs|
+        if value_or_lambda.respond_to?(:call)
+          value_or_lambda.call(*args, **kwargs)
+        else
+          value_or_lambda
+        end
+      end
+      yield
+    ensure
+      klass.define_singleton_method(method_name, original)
+    end
   end
 end