diff --git a/Gemfile b/Gemfile index c1e1ea4..6e1b32d 100644 --- a/Gemfile +++ b/Gemfile @@ -41,8 +41,7 @@ gem 'mini_magick' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false -# Structured logging for production -gem 'lograge' + # Deploy this application anywhere as a Docker container [https://kamal-deploy.org] gem 'kamal', require: false @@ -124,6 +123,10 @@ gem 'will_paginate-foundation' # Full-text search in PostgreSQL gem 'pg_search' +# Nearest neighbor search for PostgreSQL using the pgvector extension +# https://github.com/ankane/neighbor +gem 'neighbor' + # Use SLIM as our templating language gem 'slim-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 202b1e2..9cf75ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -224,11 +224,6 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.7.0) - lograge (0.14.0) - actionpack (>= 4) - activesupport (>= 4) - railties (>= 4) - request_store (~> 1.0) loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -255,6 +250,8 @@ GEM minitest (>= 5.0, < 7) ruby-progressbar msgpack (1.8.0) + neighbor (1.2.0) + activerecord (>= 7.2) nenv (0.3.0) net-http (0.9.1) uri (>= 0.11.1) @@ -427,8 +424,6 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) - request_store (1.7.0) - rack (>= 1.4) responders (3.2.0) actionpack (>= 7.0) railties (>= 7.0) @@ -585,10 +580,10 @@ DEPENDENCIES jbuilder kamal listen (~> 3.5) - lograge mini_magick minitest-rails minitest-reporters + neighbor omniauth omniauth-rails_csrf_protection omniauth_openid_connect diff --git a/README.md b/README.md index 55facc4..deaef3a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,30 @@ https://salsa.debian.org/debian/debshots/-/issues Check out the doc/README.Development.md +## Package search + +The package search combines several strategies, tried in this order: + +1. **Exact package name**: Searching for `vim` shows exactly this + package without calling any external service. +2. **Name prefix matches**: Packages whose name starts with the query + rank first, e.g. searching for `sqlite` promotes `sqlite3`, + `sqlitebrowser` and friends above all other results. +3. **Compound word splitting**: Packages whose name contains *all* + query tokens match too, so `sqlite browser` finds `sqlitebrowser`. +4. **Semantic search**: The query is embedded via the vector service + (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among + the package description embeddings are returned, using the pgvector + extension with an HNSW cosine index. Embeddings are computed from + the package description and long description and can be (re-)built + with `bin/rails debshots:compute_vectors`. +5. **Full-text fallback**: If the vector service cannot be reached, + classic PostgreSQL full-text search takes over. + +Queries that share no vocabulary with the package data (gibberish, +keyboard mash) return an honest empty result instead of random +semantic matches. + ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index b242467..726f4f0 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -362,7 +362,7 @@ class PackagesController < ApplicationController # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" - packages = packages.general_search(params[:search]) + packages = search_packages(params[:search]) end case params[:show] @@ -378,6 +378,51 @@ class PackagesController < ApplicationController packages end + # Text search for packages: an exact package name match wins without + # touching the vector service. Otherwise packages whose name starts + # with the query (or contains all query tokens, catching compound + # words like "sqlite browser" -> "sqlitebrowser") are promoted above + # the semantic (vector) nearest neighbor results. Classic PostgreSQL + # full-text search kicks in when the vector service cannot be reached. + MAX_PROMOTED_NAME_MATCHES = 8 + MAX_SEMANTIC_RESULTS = 100 + private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS + + def search_packages(query) + exact_match = Package.find_by(name: query) + return Package.where(id: exact_match.id) if exact_match + + tokens = query.to_s.split(/\s+/) + name_matches = ( + Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) + + Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES) + ).uniq(&:id) + + results = (name_matches + semantic_results(query)).uniq(&:id) + + # Queries that share no vocabulary with our package data can produce + # no meaningful results. Skip the vector service round-trip and show + # an honest empty result instead of random packages. + results.empty? ? Package.none : results + end + + # Semantic (vector) nearest neighbor search with a full-text fallback + # for when the external embedding service is unreachable. + def semantic_results(query) + unless Package.lexical_overlap?(query) + logger.debug "Query '#{query}' has no lexical overlap with package data" + return [] + end + + begin + Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a + rescue Vectorizer::Error => e + logger.error "Semantic search failed: #{e.message}" + flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' + Package.general_search(query).limit(100).to_a + end + end + # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/models/package.rb b/app/models/package.rb index 441f5c4..e2e1b00 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open() require 'json' require 'deb_importer' +# Search and embedding logic keeps accumulating here; the class is +# cohesive enough that splitting it is not worth the indirection. +# rubocop:disable Metrics/ClassLength class Package < ApplicationRecord # PostgreSQL-based full-text search: # https://github.com/Casecommons/pg_search @@ -26,6 +29,11 @@ class Package < ApplicationRecord inverse_of: :package, dependent: :destroy + # Nearest neighbor search using pgvector. The "embedding" column stores + # a vector computed from description + long_description by an external + # web service (see Vectorizer and lib/tasks/vectorize_packages.rake). + has_neighbors :embedding + # default_scope { # order('name ASC') # } @@ -40,6 +48,94 @@ class Package < ApplicationRecord long_description.split(/\n\.\n/).first if long_description end + # The text that the embedding vector is computed from: description + # and long description combined. + def embedding_text + [description, long_description].compact.join("\n") + end + + # Compute and store the embedding vector for this package by asking + # the external vector service. Raises Vectorizer::Error on failure. + # Returns true if the vector was saved. + def update_embedding! + text = embedding_text + return false if text.blank? + + self.embedding = Vectorizer.embed(text) + save! + end + + # Return a relation of packages whose stored embeddings are closest to + # the given query vector (an array of floats). Records get a + # "neighbor_distance" attribute added (cosine distance, lower is better). + # The returned relation can still be chained (e.g. filtered or paginated). + # + # Only returns packages that already have an embedding computed. + def self.nearest_to_vector(vector, limit: nil) + neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine') + limit ? neighbors.limit(limit) : neighbors + end + + # Turn a free-text search string into a vector via the external web + # service and find the closest packages. + def self.nearest_to_text(text, limit: nil) + return none if text.blank? + + nearest_to_vector(Vectorizer.embed(text), limit: limit) + end + + # Packages whose name starts with the given string - used to promote + # obvious matches like "sqlite3" for a "sqlite" search. + def self.name_starts_with(text) + return none if text.blank? + + where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name) + end + + # Packages whose name contains all given tokens as substrings. Catches + # compound words split by the user, e.g. "sqlite browser" finding the + # "sqlitebrowser" package. + def self.name_contains_all(tokens) + tokens = tokens.select { |token| token.length >= 2 } + return none if tokens.empty? + + scope = self + tokens.each do |token| + scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\') + end + scope.order(:name) + end + + def self.escape_for_like(text) + # Escape LIKE wildcards so that package name characters like "+" or "." + # cannot break the pattern + text.gsub(/[\\%_]/) { |char| "\\#{char}" } + end + + # Check whether the given text shares at least one lexeme with any + # package name/description in the database. Queries without such an + # overlap (gibberish, keyboard mash, unsupported languages) can + # neither match full-text nor produce meaningful semantic results - + # the embedding model maps unknown tokens close to the corpus + # centroid, making them look like good matches. + def self.lexical_overlap?(text) + tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) } + return false if tokens.empty? + + # Pin the same text search dictionary that the packages_fts index + # uses - the server-wide default can be anything. + tsquery = tokens.map do |token| + sanitize_sql(["plainto_tsquery('english', ?)", token]) + end.join(' || ') + # The expression must match the packages_fts GIN index definition + # so that Postgres can use the index. + where( + "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \ + "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \ + "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})" + ).exists? + end + # Return a query of all packages that have screenshots def self.with_screenshots # Query for all packages who's ID appears in a screenshot's "package_id" field diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 84d8599..53009da 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -14,7 +14,7 @@ - if @packages.any? - if @view_style==:grid .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium" - - @packages.all.each do |pkg| + - @packages.each do |pkg| .cell.pkgcard data-equalizer-watch=true a.black href=package_path(name: pkg.name) .image diff --git a/config/initializers/will_paginate.rb b/config/initializers/will_paginate.rb new file mode 100644 index 0000000..e36b7b3 --- /dev/null +++ b/config/initializers/will_paginate.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Enable will_paginate's Array#paginate so that merged search results +# (name prefix matches + semantic results) can be paginated like +# ActiveRecord relations. +require 'will_paginate/array' diff --git a/db/migrate/20260821203833_install_neighbor_vector.rb b/db/migrate/20260821203833_install_neighbor_vector.rb new file mode 100644 index 0000000..d591f1c --- /dev/null +++ b/db/migrate/20260821203833_install_neighbor_vector.rb @@ -0,0 +1,5 @@ +class InstallNeighborVector < ActiveRecord::Migration[8.1] + def change + enable_extension "vector" + end +end diff --git a/db/migrate/20260821210000_add_embedding_to_packages.rb b/db/migrate/20260821210000_add_embedding_to_packages.rb new file mode 100644 index 0000000..62825f9 --- /dev/null +++ b/db/migrate/20260821210000_add_embedding_to_packages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Adds a vector column to store text embeddings of the package +# description and long description. The vectors are computed by an +# external web service and used for nearest neighbor search. +# +# The pgvector extension is enabled by the InstallNeighborVector migration. +class AddEmbeddingToPackages < ActiveRecord::Migration[8.1] + def change + add_column :packages, :embedding, :vector, limit: 384 + + # HNSW index for fast approximate nearest neighbor searches + add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops + end +end diff --git a/db/schema.rb b/db/schema.rb index 2c57b69..a2ad488 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,133 +10,131 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do - # These are extensions that must be enabled to support this database - enable_extension 'pg_catalog.plpgsql' +ActiveRecord::Schema[8.1].define(version: 2026_08_21_210000) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + enable_extension "vector" - create_table 'action_mailbox_inbound_emails', force: :cascade do |t| - t.integer 'status', default: 0, null: false - t.string 'message_id', null: false - t.string 'message_checksum', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['message_id', 'message_checksum'], -name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true + create_table "action_mailbox_inbound_emails", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "message_checksum", null: false + t.string "message_id", null: false + t.integer "status", default: 0, null: false + t.datetime "updated_at", null: false + t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true end - create_table 'action_text_rich_texts', force: :cascade do |t| - t.string 'name', null: false - t.text 'body' - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', -unique: true + create_table "action_text_rich_texts", force: :cascade do |t| + t.text "body" + t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.datetime "updated_at", null: false + t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true end - create_table 'active_storage_attachments', force: :cascade do |t| - t.string 'name', null: false - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.bigint 'blob_id', null: false - t.datetime 'created_at', precision: nil, null: false - t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id' - t.index ['record_type', 'record_id', 'name', 'blob_id'], -name: 'index_active_storage_attachments_uniqueness', unique: true + create_table "active_storage_attachments", force: :cascade do |t| + t.bigint "blob_id", null: false + t.datetime "created_at", precision: nil, null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end - create_table 'active_storage_blobs', force: :cascade do |t| - t.string 'key', null: false - t.string 'filename', null: false - t.string 'content_type' - t.text 'metadata' - t.bigint 'byte_size', null: false - t.string 'checksum' - t.datetime 'created_at', precision: nil, null: false - t.string 'service_name', null: false - t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true + create_table "active_storage_blobs", force: :cascade do |t| + t.bigint "byte_size", null: false + t.string "checksum" + t.string "content_type" + t.datetime "created_at", precision: nil, null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" + t.string "service_name", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end - create_table 'active_storage_variant_records', force: :cascade do |t| - t.bigint 'blob_id', null: false - t.string 'variation_digest', null: false - t.index ['blob_id', 'variation_digest'], -name: 'index_active_storage_variant_records_uniqueness', unique: true + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end - create_table 'logs', id: :serial, force: :cascade do |t| - t.string 'message' - t.string 'level' - t.string 'section' - t.inet 'ip_address' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.integer 'user_id' - t.integer 'package_id' - t.integer 'screenshot_id' + create_table "logs", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.inet "ip_address" + t.string "level" + t.string "message" + t.integer "package_id" + t.integer "screenshot_id" + t.string "section" + t.datetime "updated_at", precision: nil, null: false + t.integer "user_id" end - create_table 'packages', id: :serial, force: :cascade do |t| - t.string 'name', null: false - t.string 'description', limit: 80 - t.string 'section', limit: 50 - t.string 'maintainer', limit: 100 - t.string 'maintainer_email', limit: 100 - t.string 'homepage', limit: 400 - t.string 'version', limit: 200 - t.text 'long_description' - t.string 'origin', limit: 80 - t.datetime 'created_at', precision: nil - t.datetime 'updated_at', precision: nil - t.integer 'visits', default: 0 - t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", -name: 'packages_fts', using: :gin - t.unique_constraint ['name'], name: 'packages_name_key' + create_table "packages", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil + t.string "description", limit: 80 + t.vector "embedding", limit: 384 + t.string "homepage", limit: 400 + t.text "long_description" + t.string "maintainer", limit: 100 + t.string "maintainer_email", limit: 100 + t.string "name", null: false + t.string "origin", limit: 80 + t.string "section", limit: 50 + t.datetime "updated_at", precision: nil + t.string "version", limit: 200 + t.integer "visits", default: 0 + t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin + t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw + t.unique_constraint ["name"], name: "packages_name_key" end - create_table 'screenshots', id: :serial, force: :cascade do |t| - t.integer 'package_id' - t.string 'version', limit: 50 - t.datetime 'created_at', precision: nil - t.string 'uploaderhash', limit: 72 - t.boolean 'approved', default: false, null: false - t.text 'description' - t.datetime 'updated_at', precision: nil - t.string 'image_fingerprint' - t.integer 'user_id', default: 0 - t.text 'simage_data' - t.inet 'uploaderip' - t.boolean 'hidden', default: false - t.index ['id', 'approved'], name: 'id_approved' - t.index ['id', 'uploaderhash'], name: 'id_uploaderhash' + create_table "screenshots", id: :serial, force: :cascade do |t| + t.boolean "approved", default: false, null: false + t.datetime "created_at", precision: nil + t.text "description" + t.boolean "hidden", default: false + t.string "image_fingerprint" + t.integer "package_id" + t.text "simage_data" + t.datetime "updated_at", precision: nil + t.string "uploaderhash", limit: 72 + t.inet "uploaderip" + t.integer "user_id", default: 0 + t.string "version", limit: 50 + t.index ["id", "approved"], name: "id_approved" + t.index ["id", "uploaderhash"], name: "id_uploaderhash" end - create_table 'users', id: :serial, force: :cascade do |t| - t.text 'name' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.string 'email', default: '', null: false - t.string 'encrypted_password', default: '', null: false - t.integer 'sign_in_count', default: 0, null: false - t.datetime 'current_sign_in_at', precision: nil - t.datetime 'last_sign_in_at', precision: nil - t.inet 'current_sign_in_ip' - t.inet 'last_sign_in_ip' - t.integer 'failed_attempts', default: 0, null: false - t.string 'unlock_token' - t.datetime 'locked_at', precision: nil - t.string 'provider' - t.string 'uid' - t.boolean 'admin_role', default: false - t.boolean 'moderator_role', default: false - t.boolean 'pseudo', default: false - t.integer 'approved_screenshots', default: 0 - t.integer 'rejected_screenshots', default: 0 - t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true + create_table "users", id: :serial, force: :cascade do |t| + t.boolean "admin_role", default: false + t.integer "approved_screenshots", default: 0 + t.datetime "created_at", precision: nil, null: false + t.datetime "current_sign_in_at", precision: nil + t.inet "current_sign_in_ip" + t.string "email", default: "", null: false + t.string "encrypted_password", default: "", null: false + t.integer "failed_attempts", default: 0, null: false + t.datetime "last_sign_in_at", precision: nil + t.inet "last_sign_in_ip" + t.datetime "locked_at", precision: nil + t.boolean "moderator_role", default: false + t.text "name" + t.string "provider" + t.boolean "pseudo", default: false + t.integer "rejected_screenshots", default: 0 + t.integer "sign_in_count", default: 0, null: false + t.string "uid" + t.string "unlock_token" + t.datetime "updated_at", precision: nil, null: false + t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true end - add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey' + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey" end diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake new file mode 100644 index 0000000..1c19837 --- /dev/null +++ b/lib/tasks/vectorize_packages.rake @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +namespace :debshots do + desc 'Compute embedding vectors for all packages based on their description ' \ + 'and long description (missing ones only unless FORCE=1). ' \ + 'Set CONCURRENCY=n to control parallel HTTP requests.' + task compute_vectors: :environment do + logger = Logger.new($stdout) + logger.level = Logger::INFO + + force = ENV['FORCE'].present? + concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max + + # Never use more threads than the connection pool has connections. + # Keep one connection reserved for the main thread. + pool_size = ActiveRecord::Base.connection_pool.size + concurrency = [concurrency, pool_size - 1].min + concurrency = 0 if concurrency.negative? + + scope = force ? Package.all : Package.where(embedding: nil) + total = scope.count + + if total.zero? + logger.info 'No packages need vector computation.' + next + end + + logger.info "Computing vectors for #{total} packages " \ + "(#{force ? 'recomputing' : 'missing only'}, " \ + "service: #{Vectorizer.service_url}, " \ + "concurrency: #{concurrency.zero? ? 1 : concurrency})" + + stats_mutex = Mutex.new + stats = { done: 0, failed: 0 } + + process_package = lambda do |package| + stats_mutex.synchronize do + begin + raise ArgumentError, 'no description text' if package.embedding_text.blank? + + package.update_embedding! + stats[:done] += 1 + rescue Vectorizer::Error, ArgumentError => e + stats[:failed] += 1 + logger.error "Failed for package #{package.name}: #{e.message}" + end + + processed = stats[:done] + stats[:failed] + return unless (processed % 100).zero? + + percent = (processed.to_f / total * 100).round(1) + logger.info "Progress: #{processed}/#{total} (#{percent}%)" + end + end + + if concurrency.zero? + # Sequential fallback when the connection pool has no room for workers + scope.find_each { |package| process_package.call(package) } + else + # Feed package IDs into a queue that the worker threads pull from. + work_queue = Queue.new + scope.pluck(:id).each { |id| work_queue << id } + concurrency.times { work_queue << nil } # one poison pill per worker + + workers = Array.new(concurrency) do + Thread.new do + # Each thread gets its own database connection from the pool. + ActiveRecord::Base.connection_pool.with_connection do + while (package_id = work_queue.pop) + package = Package.find_by(id: package_id) + process_package.call(package) if package + end + end + rescue StandardError => e + logger.error "Worker crashed: #{e.message}" + end + end + + workers.each(&:join) + end + + logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures." + end +end diff --git a/lib/vectorizer.rb b/lib/vectorizer.rb new file mode 100644 index 0000000..84df3c4 --- /dev/null +++ b/lib/vectorizer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' + +# Client for the external web service that turns text into embedding +# vectors for nearest neighbor search (pgvector). +# +# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...] +# +# The service URL can be overridden with the VECTOR_SERVICE_URL +# environment variable. +class Vectorizer + # Raised when the vector service cannot be reached or returns garbage + class Error < StandardError; end + + DEFAULT_URL = 'https://vector-search.workaround.org/vector' + + class << self + # Compute the embedding vector for the given text. + # Returns an array of floats. + def embed(text) + new.embed(text) + end + + def service_url + ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL) + end + end + + def embed(text) + response = post_text(text) + + unless response.is_a?(Net::HTTPSuccess) + raise Error, "Vector service returned HTTP #{response.code}" + end + + JSON.parse(response.body).fetch('vector') + rescue JSON::ParserError => e + raise Error, "Vector service returned invalid JSON: #{e.message}" + rescue KeyError + raise Error, 'Vector service response did not contain a "vector" field' + end + + private + + def post_text(text) + uri = URI.parse(self.class.service_url) + request = build_request(uri, text) + + options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 } + Net::HTTP.start(uri.host, uri.port, **options) do |http| + http.request(request) + end + rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e + raise Error, "Could not reach vector service: #{e.message}" + end + + def build_request(uri, text) + request = Net::HTTP::Post.new(uri.request_uri) + request['Content-Type'] = 'application/json' + request.body = { text: text }.to_json + + # Optional Bearer token if the service runs with API_SECRET set + secret = ENV['VECTOR_SERVICE_SECRET'] + request['Authorization'] = "Bearer #{secret}" if secret.present? + + request + end +end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index c9eae31..4d89d88 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class PackagesControllerTest < ActionController::TestCase @@ -20,6 +22,81 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'A program to browse web sites.' end + test 'exact package name search wins without calling the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'vim' } + assert_response :success + assert_select 'div', 'vim' + end + end + + test 'gibberish search shows no results and never calls the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'asdfgh zzzz' } + assert_response :success + # the grid renders every package in the database when no search + # filter applies - so an empty result must not fall back to that + assert_select 'div', text: /package-\d+/, count: 0 + end + end + + test 'name prefix matches rank above semantic results' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do + get :grid, params: { search: 'package' } + assert_response :success + # fixtures create package-0..package-99 - prefix hits must render + assert_select 'div', 'package-0' + # ...and before any semantically found package + assert response.body.index('package-0') < response.body.index('vim') + end + end + + test 'prefix matches and semantic results are deduplicated' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do + get :grid, params: { search: 'fire' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + end + end + + test 'compound words in search match packages with joined names' do + # "fire fox" should find "firefox" via name substring matching + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.none }) do + get :grid, params: { search: 'fire fox' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + end + end + + test 'search uses semantic (vector) results' do + relation = Package.where(name: packages(:vim).name) + with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + end + end + + # There is deliberately no fallback for an empty vector search + # result: KNN always returns something as long as packages have + # embeddings. Full-text search is only used when the vector service + # cannot be reached. + + test 'search falls back to full-text search when vector service fails' do + with_stubbed_method(Vectorizer, :embed, + ->(_text) { raise Vectorizer::Error, 'service down' }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + assert flash[:error].present? + end + end + # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') diff --git a/test/models/package_test.rb b/test/models/package_test.rb index 124f6c4..36562a2 100644 --- a/test/models/package_test.rb +++ b/test/models/package_test.rb @@ -1,7 +1,107 @@ +# frozen_string_literal: true + require 'test_helper' class PackageTest < ActiveSupport::TestCase - # test "the truth" do - # assert true - # end + test 'embedding_text combines description and long description' do + package = packages(:firefox) + assert_equal "A web browser\nA program to browse web sites.", package.embedding_text + end + + test 'embedding_text works with missing long description' do + package = packages(:firefox) + package.long_description = nil + assert_equal 'A web browser', package.embedding_text + end + + test 'update_embedding! stores the vector returned by the vector service' do + vector = Array.new(384) { |i| i / 384.0 } + with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do + package = packages(:vim) + assert package.update_embedding! + # The vector type stores float32 so allow small rounding differences + package.reload.embedding.to_a.each_with_index do |actual, i| + assert_in_delta vector[i], actual, 1e-6 + end + end + end + + test 'update_embedding! does nothing without description text' do + package = packages(:firefox) + package.description = nil + package.long_description = nil + with_stubbed_method(Vectorizer, :embed, + ->(_text) { flunk 'should not call the vector service' }) do + assert_not package.update_embedding! + end + end + + test 'nearest_to_text returns none for blank text' do + assert_empty Package.nearest_to_text('') + assert_empty Package.nearest_to_text(nil) + end + + test 'name_starts_with matches name prefixes case-insensitively' do + assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name + assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name + assert_empty Package.name_starts_with('') + assert_empty Package.name_starts_with(nil) + end + + test 'name_contains_all finds compound words split by spaces' do + assert_includes Package.name_contains_all(%w[fire fox]).map(&:name), + packages(:firefox).name + # all tokens must be present in the name + assert_empty Package.name_contains_all(%w[firefox zzzz]) + assert_empty Package.name_contains_all([]) + # single character tokens would flood results - ignore them + assert_empty Package.name_contains_all(['f']) + end + + test 'lexical_overlap? pins the english dictionary regardless of server config' do + # Regression: with default_text_search_config = german (as on some + # servers) 'browser' stems to 'brows' which never matches the + # english-stemmed index. + connection = ActiveRecord::Base.connection + original = connection.select_value('SHOW default_text_search_config') + connection.execute("SET default_text_search_config = 'german'") + assert Package.lexical_overlap?('browser') + ensure + connection&.execute("SET default_text_search_config = '#{original}'") + end + + test 'lexical_overlap? detects real words vs gibberish' do + # words present in fixture package descriptions + assert Package.lexical_overlap?('browser') + assert Package.lexical_overlap?('email client for vim syntax') + assert Package.lexical_overlap?('the browser') # stopwords are ignored + # gibberish, numbers and empty strings share no vocabulary + assert_not Package.lexical_overlap?('asdfgh zzzz') + assert_not Package.lexical_overlap?('123 !!!') + assert_not Package.lexical_overlap?('the') # stopword-only query + assert_not Package.lexical_overlap?('') + assert_not Package.lexical_overlap?(nil) + end + + test 'nearest_to_text embeds the query and searches' do + vector = Array.new(384, 0.25) + embed_called_with = nil + with_stubbed_method(Vectorizer, :embed, + lambda { |text| + embed_called_with = text + vector + }) do + received = [] + results = with_stubbed_method(Package, :nearest_neighbors, + lambda { |_field, received_vector, **_opts| + received << received_vector + [packages(:vim)] + }) do + Package.nearest_to_text('some search text') + end + assert_equal [packages(:vim)], results + assert_equal 'some search text', embed_called_with + assert_equal [vector], received + end + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 4889a19..1d786a4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) require 'rails/test_help' @@ -15,6 +17,23 @@ module ActiveSupport class TestCase # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. fixtures :all + + # Replace a class-level method with a fixed return value or lambda + # for the duration of the block. Used to keep tests away from + # external services like the vector embedding web service. + def with_stubbed_method(klass, method_name, value_or_lambda) + original = klass.method(method_name) + klass.define_singleton_method(method_name) do |*args, **kwargs| + if value_or_lambda.respond_to?(:call) + value_or_lambda.call(*args, **kwargs) + else + value_or_lambda + end + end + yield + ensure + klass.define_singleton_method(method_name, original) + end end end