Christoph Haas
4948fe50e3
Package text search now combines several strategies: exact name match,
name prefix promotion, compound word splitting ("sqlite browser" finds
"sqlitebrowser") and semantic nearest neighbor search over description
embeddings computed by an external embedding service (all-MiniLM-L6-v2,
384 dims) stored with the pgvector extension. Classic PostgreSQL
full-text search remains as fallback when the vector service is
unreachable. Gibberish queries without lexical overlap with the package
data return empty results instead of random matches.
Embeddings can be backfilled with bin/rails debshots:compute_vectors.
Also drops the unused lograge gem from the Gemfile.
107 lines
4.3 KiB
Ruby
107 lines
4.3 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require 'test_helper'
|
|
|
|
class PackageTest < ActiveSupport::TestCase
|
|
test 'embedding_text combines description and long description' do
|
|
package = packages(:firefox)
|
|
assert_equal "A web browser\nA program to browse web sites.", package.embedding_text
|
|
end
|
|
|
|
test 'embedding_text works with missing long description' do
|
|
package = packages(:firefox)
|
|
package.long_description = nil
|
|
assert_equal 'A web browser', package.embedding_text
|
|
end
|
|
|
|
test 'update_embedding! stores the vector returned by the vector service' do
|
|
vector = Array.new(384) { |i| i / 384.0 }
|
|
with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do
|
|
package = packages(:vim)
|
|
assert package.update_embedding!
|
|
# The vector type stores float32 so allow small rounding differences
|
|
package.reload.embedding.to_a.each_with_index do |actual, i|
|
|
assert_in_delta vector[i], actual, 1e-6
|
|
end
|
|
end
|
|
end
|
|
|
|
test 'update_embedding! does nothing without description text' do
|
|
package = packages(:firefox)
|
|
package.description = nil
|
|
package.long_description = nil
|
|
with_stubbed_method(Vectorizer, :embed,
|
|
->(_text) { flunk 'should not call the vector service' }) do
|
|
assert_not package.update_embedding!
|
|
end
|
|
end
|
|
|
|
test 'nearest_to_text returns none for blank text' do
|
|
assert_empty Package.nearest_to_text('')
|
|
assert_empty Package.nearest_to_text(nil)
|
|
end
|
|
|
|
test 'name_starts_with matches name prefixes case-insensitively' do
|
|
assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name
|
|
assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name
|
|
assert_empty Package.name_starts_with('')
|
|
assert_empty Package.name_starts_with(nil)
|
|
end
|
|
|
|
test 'name_contains_all finds compound words split by spaces' do
|
|
assert_includes Package.name_contains_all(%w[fire fox]).map(&:name),
|
|
packages(:firefox).name
|
|
# all tokens must be present in the name
|
|
assert_empty Package.name_contains_all(%w[firefox zzzz])
|
|
assert_empty Package.name_contains_all([])
|
|
# single character tokens would flood results - ignore them
|
|
assert_empty Package.name_contains_all(['f'])
|
|
end
|
|
|
|
test 'lexical_overlap? pins the english dictionary regardless of server config' do
|
|
# Regression: with default_text_search_config = german (as on some
|
|
# servers) 'browser' stems to 'brows' which never matches the
|
|
# english-stemmed index.
|
|
connection = ActiveRecord::Base.connection
|
|
original = connection.select_value('SHOW default_text_search_config')
|
|
connection.execute("SET default_text_search_config = 'german'")
|
|
assert Package.lexical_overlap?('browser')
|
|
ensure
|
|
connection&.execute("SET default_text_search_config = '#{original}'")
|
|
end
|
|
|
|
test 'lexical_overlap? detects real words vs gibberish' do
|
|
# words present in fixture package descriptions
|
|
assert Package.lexical_overlap?('browser')
|
|
assert Package.lexical_overlap?('email client for vim syntax')
|
|
assert Package.lexical_overlap?('the browser') # stopwords are ignored
|
|
# gibberish, numbers and empty strings share no vocabulary
|
|
assert_not Package.lexical_overlap?('asdfgh zzzz')
|
|
assert_not Package.lexical_overlap?('123 !!!')
|
|
assert_not Package.lexical_overlap?('the') # stopword-only query
|
|
assert_not Package.lexical_overlap?('')
|
|
assert_not Package.lexical_overlap?(nil)
|
|
end
|
|
|
|
test 'nearest_to_text embeds the query and searches' do
|
|
vector = Array.new(384, 0.25)
|
|
embed_called_with = nil
|
|
with_stubbed_method(Vectorizer, :embed,
|
|
lambda { |text|
|
|
embed_called_with = text
|
|
vector
|
|
}) do
|
|
received = []
|
|
results = with_stubbed_method(Package, :nearest_neighbors,
|
|
lambda { |_field, received_vector, **_opts|
|
|
received << received_vector
|
|
[packages(:vim)]
|
|
}) do
|
|
Package.nearest_to_text('some search text')
|
|
end
|
|
assert_equal [packages(:vim)], results
|
|
assert_equal 'some search text', embed_called_with
|
|
assert_equal [vector], received
|
|
end
|
|
end
|
|
end
|