Add semantic package search via pgvector embeddings

Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.

Embeddings can be backfilled with bin/rails debshots:compute_vectors.

Also drops the unused lograge gem from the Gemfile.

test/controllers/packages_controller_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackagesControllerTest < ActionController::TestCase
@@ -20,6 +22,81 @@ class PackagesControllerTest < ActionController::TestCase
     assert_select 'div', 'A program to browse web sites.'
   end
 
+  test 'exact package name search wins without calling the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'vim' }
+      assert_response :success
+      assert_select 'div', 'vim'
+    end
+  end
+
+  test 'gibberish search shows no results and never calls the vector service' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text) { flunk 'should not call vector search' }) do
+      get :grid, params: { search: 'asdfgh zzzz' }
+      assert_response :success
+      # the grid renders every package in the database when no search
+      # filter applies - so an empty result must not fall back to that
+      assert_select 'div', text: /package-\d+/, count: 0
+    end
+  end
+
+  test 'name prefix matches rank above semantic results' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do
+      get :grid, params: { search: 'package' }
+      assert_response :success
+      # fixtures create package-0..package-99 - prefix hits must render
+      assert_select 'div', 'package-0'
+      # ...and before any semantically found package
+      assert response.body.index('package-0') < response.body.index('vim')
+    end
+  end
+
+  test 'prefix matches and semantic results are deduplicated' do
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do
+      get :grid, params: { search: 'fire' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+    end
+  end
+
+  test 'compound words in search match packages with joined names' do
+    # "fire fox" should find "firefox" via name substring matching
+    with_stubbed_method(Package, :nearest_to_text,
+                        ->(_text, **_opts) { Package.none }) do
+      get :grid, params: { search: 'fire fox' }
+      assert_response :success
+      assert_select 'div', text: 'firefox', count: 1
+    end
+  end
+
+  test 'search uses semantic (vector) results' do
+    relation = Package.where(name: packages(:vim).name)
+    with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+    end
+  end
+
+  # There is deliberately no fallback for an empty vector search
+  # result: KNN always returns something as long as packages have
+  # embeddings. Full-text search is only used when the vector service
+  # cannot be reached.
+
+  test 'search falls back to full-text search when vector service fails' do
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { raise Vectorizer::Error, 'service down' }) do
+      get :grid, params: { search: 'editor' }
+      assert_response :success
+      assert_select 'div', 'vim'
+      assert flash[:error].present?
+    end
+  end
+
   # test 'should get thumbnail for a package and a desired version' do
   #   # get '/thumbnail-with-version/package-5/0.1'
   #   get thumbnail_with_version_url('package-5', '0.1')

test/models/package_test.rb

@@ -1,7 +1,107 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class PackageTest < ActiveSupport::TestCase
-  # test "the truth" do
-  #   assert true
-  # end
+  test 'embedding_text combines description and long description' do
+    package = packages(:firefox)
+    assert_equal "A web browser\nA program to browse web sites.", package.embedding_text
+  end
+
+  test 'embedding_text works with missing long description' do
+    package = packages(:firefox)
+    package.long_description = nil
+    assert_equal 'A web browser', package.embedding_text
+  end
+
+  test 'update_embedding! stores the vector returned by the vector service' do
+    vector = Array.new(384) { |i| i / 384.0 }
+    with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do
+      package = packages(:vim)
+      assert package.update_embedding!
+      # The vector type stores float32 so allow small rounding differences
+      package.reload.embedding.to_a.each_with_index do |actual, i|
+        assert_in_delta vector[i], actual, 1e-6
+      end
+    end
+  end
+
+  test 'update_embedding! does nothing without description text' do
+    package = packages(:firefox)
+    package.description = nil
+    package.long_description = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        ->(_text) { flunk 'should not call the vector service' }) do
+      assert_not package.update_embedding!
+    end
+  end
+
+  test 'nearest_to_text returns none for blank text' do
+    assert_empty Package.nearest_to_text('')
+    assert_empty Package.nearest_to_text(nil)
+  end
+
+  test 'name_starts_with matches name prefixes case-insensitively' do
+    assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name
+    assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name
+    assert_empty Package.name_starts_with('')
+    assert_empty Package.name_starts_with(nil)
+  end
+
+  test 'name_contains_all finds compound words split by spaces' do
+    assert_includes Package.name_contains_all(%w[fire fox]).map(&:name),
+                    packages(:firefox).name
+    # all tokens must be present in the name
+    assert_empty Package.name_contains_all(%w[firefox zzzz])
+    assert_empty Package.name_contains_all([])
+    # single character tokens would flood results - ignore them
+    assert_empty Package.name_contains_all(['f'])
+  end
+
+  test 'lexical_overlap? pins the english dictionary regardless of server config' do
+    # Regression: with default_text_search_config = german (as on some
+    # servers) 'browser' stems to 'brows' which never matches the
+    # english-stemmed index.
+    connection = ActiveRecord::Base.connection
+    original = connection.select_value('SHOW default_text_search_config')
+    connection.execute("SET default_text_search_config = 'german'")
+    assert Package.lexical_overlap?('browser')
+  ensure
+    connection&.execute("SET default_text_search_config = '#{original}'")
+  end
+
+  test 'lexical_overlap? detects real words vs gibberish' do
+    # words present in fixture package descriptions
+    assert Package.lexical_overlap?('browser')
+    assert Package.lexical_overlap?('email client for vim syntax')
+    assert Package.lexical_overlap?('the browser') # stopwords are ignored
+    # gibberish, numbers and empty strings share no vocabulary
+    assert_not Package.lexical_overlap?('asdfgh zzzz')
+    assert_not Package.lexical_overlap?('123 !!!')
+    assert_not Package.lexical_overlap?('the') # stopword-only query
+    assert_not Package.lexical_overlap?('')
+    assert_not Package.lexical_overlap?(nil)
+  end
+
+  test 'nearest_to_text embeds the query and searches' do
+    vector = Array.new(384, 0.25)
+    embed_called_with = nil
+    with_stubbed_method(Vectorizer, :embed,
+                        lambda { |text|
+                          embed_called_with = text
+                          vector
+                        }) do
+      received = []
+      results = with_stubbed_method(Package, :nearest_neighbors,
+                                    lambda { |_field, received_vector, **_opts|
+                                      received << received_vector
+                                      [packages(:vim)]
+                                    }) do
+        Package.nearest_to_text('some search text')
+      end
+      assert_equal [packages(:vim)], results
+      assert_equal 'some search text', embed_called_with
+      assert_equal [vector], received
+    end
+  end
 end

test/test_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 ENV['RAILS_ENV'] ||= 'test'
 require File.expand_path('../config/environment', __dir__)
 require 'rails/test_help'
@@ -15,6 +17,23 @@ module ActiveSupport
   class TestCase
     # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
     fixtures :all
+
+    # Replace a class-level method with a fixed return value or lambda
+    # for the duration of the block. Used to keep tests away from
+    # external services like the vector embedding web service.
+    def with_stubbed_method(klass, method_name, value_or_lambda)
+      original = klass.method(method_name)
+      klass.define_singleton_method(method_name) do |*args, **kwargs|
+        if value_or_lambda.respond_to?(:call)
+          value_or_lambda.call(*args, **kwargs)
+        else
+          value_or_lambda
+        end
+      end
+      yield
+    ensure
+      klass.define_singleton_method(method_name, original)
+    end
   end
 end