Add semantic package search via pgvector embeddings
Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.
Embeddings can be backfilled with bin/rails debshots:compute_vectors.
Also drops the unused lograge gem from the Gemfile.
This commit is contained in:
parent
c7e8109bbd
commit
4948fe50e3
15 changed files with 662 additions and 125 deletions
|
|
@ -1,3 +1,5 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
require 'test_helper'
|
||||
|
||||
class PackagesControllerTest < ActionController::TestCase
|
||||
|
|
@ -20,6 +22,81 @@ class PackagesControllerTest < ActionController::TestCase
|
|||
assert_select 'div', 'A program to browse web sites.'
|
||||
end
|
||||
|
||||
test 'exact package name search wins without calling the vector service' do
|
||||
with_stubbed_method(Package, :nearest_to_text,
|
||||
->(_text) { flunk 'should not call vector search' }) do
|
||||
get :grid, params: { search: 'vim' }
|
||||
assert_response :success
|
||||
assert_select 'div', 'vim'
|
||||
end
|
||||
end
|
||||
|
||||
test 'gibberish search shows no results and never calls the vector service' do
|
||||
with_stubbed_method(Package, :nearest_to_text,
|
||||
->(_text) { flunk 'should not call vector search' }) do
|
||||
get :grid, params: { search: 'asdfgh zzzz' }
|
||||
assert_response :success
|
||||
# the grid renders every package in the database when no search
|
||||
# filter applies - so an empty result must not fall back to that
|
||||
assert_select 'div', text: /package-\d+/, count: 0
|
||||
end
|
||||
end
|
||||
|
||||
test 'name prefix matches rank above semantic results' do
|
||||
with_stubbed_method(Package, :nearest_to_text,
|
||||
->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do
|
||||
get :grid, params: { search: 'package' }
|
||||
assert_response :success
|
||||
# fixtures create package-0..package-99 - prefix hits must render
|
||||
assert_select 'div', 'package-0'
|
||||
# ...and before any semantically found package
|
||||
assert response.body.index('package-0') < response.body.index('vim')
|
||||
end
|
||||
end
|
||||
|
||||
test 'prefix matches and semantic results are deduplicated' do
|
||||
with_stubbed_method(Package, :nearest_to_text,
|
||||
->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do
|
||||
get :grid, params: { search: 'fire' }
|
||||
assert_response :success
|
||||
assert_select 'div', text: 'firefox', count: 1
|
||||
end
|
||||
end
|
||||
|
||||
test 'compound words in search match packages with joined names' do
|
||||
# "fire fox" should find "firefox" via name substring matching
|
||||
with_stubbed_method(Package, :nearest_to_text,
|
||||
->(_text, **_opts) { Package.none }) do
|
||||
get :grid, params: { search: 'fire fox' }
|
||||
assert_response :success
|
||||
assert_select 'div', text: 'firefox', count: 1
|
||||
end
|
||||
end
|
||||
|
||||
test 'search uses semantic (vector) results' do
|
||||
relation = Package.where(name: packages(:vim).name)
|
||||
with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do
|
||||
get :grid, params: { search: 'editor' }
|
||||
assert_response :success
|
||||
assert_select 'div', 'vim'
|
||||
end
|
||||
end
|
||||
|
||||
# There is deliberately no fallback for an empty vector search
|
||||
# result: KNN always returns something as long as packages have
|
||||
# embeddings. Full-text search is only used when the vector service
|
||||
# cannot be reached.
|
||||
|
||||
test 'search falls back to full-text search when vector service fails' do
|
||||
with_stubbed_method(Vectorizer, :embed,
|
||||
->(_text) { raise Vectorizer::Error, 'service down' }) do
|
||||
get :grid, params: { search: 'editor' }
|
||||
assert_response :success
|
||||
assert_select 'div', 'vim'
|
||||
assert flash[:error].present?
|
||||
end
|
||||
end
|
||||
|
||||
# test 'should get thumbnail for a package and a desired version' do
|
||||
# # get '/thumbnail-with-version/package-5/0.1'
|
||||
# get thumbnail_with_version_url('package-5', '0.1')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue