diff --git a/Dockerfile b/Dockerfile index 177771d..e451fa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,11 @@ ARG RUBY_VERSION=3.4.7 ARG NODE_VERSION=22.16.0 ARG BUN_VERSION=1.3.9 +ARG GIT_REV=unknown # --- Stage 1: Grab Node.js Binaries --- # We use the official image as a source for pre-compiled binaries -FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source +FROM docker.io/library/node:${NODE_VERSION} AS node_source # --- Stage 2: Base (Runtime Environment) --- FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base @@ -36,15 +37,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ apt-get install --no-install-recommends -y \ - build-essential git libyaml-dev node-gyp pkg-config python-is-python3 + build-essential git libyaml-dev pkg-config python-is-python3 -# "Heist" Node.js and npm from the node_source stage (build-time only) -COPY --from=node_source /usr/local/bin/node /usr/local/bin/node -COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules +# Install Node.js and npm from node_source +COPY --from=node_source /usr/local/ /usr/local/ -# Re-link npm and install Bun (build-time only) -RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ - npm install -g bun@${BUN_VERSION} +# Install Bun +RUN npm install -g bun@${BUN_VERSION} # 1. Install Gems (Separate COPY for maximum caching) COPY Gemfile Gemfile.lock ./ @@ -80,8 +79,11 @@ RUN groupadd --system --gid 1000 rails && \ useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \ chown -R rails:rails db log tmp public -# Create versioninfo with build timestamp -RUN date +"%Y-%m-%d %H:%M:%S %Z" > public/versioninfo +# Redeclare so it is in scope in this final stage +ARG GIT_REV=unknown + +# Create versioninfo with git revision and build timestamp +RUN printf 'git-rev: %s\nbuilt-at: %s\n' "$GIT_REV" "$(date +'%Y-%m-%d %H:%M:%S %Z')" > public/versioninfo USER 1000:1000 diff --git a/Gemfile b/Gemfile index c1e1ea4..c7a6922 100644 --- a/Gemfile +++ b/Gemfile @@ -41,8 +41,7 @@ gem 'mini_magick' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false -# Structured logging for production -gem 'lograge' + # Deploy this application anywhere as a Docker container [https://kamal-deploy.org] gem 'kamal', require: false @@ -109,6 +108,8 @@ group :test do # Use system testing [https://guides.rubyonrails.org/testing.html#system-testing] gem 'capybara' gem 'selenium-webdriver' + # Provides the assigns() test helper extracted from modern Rails + gem 'rails-controller-testing' end # TODO… https://github.com/galetahub/simple-captcha @@ -124,6 +125,10 @@ gem 'will_paginate-foundation' # Full-text search in PostgreSQL gem 'pg_search' +# Nearest neighbor search for PostgreSQL using the pgvector extension +# https://github.com/ankane/neighbor +gem 'neighbor' + # Use SLIM as our templating language gem 'slim-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 202b1e2..65bbd8a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -224,11 +224,6 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.7.0) - lograge (0.14.0) - actionpack (>= 4) - activesupport (>= 4) - railties (>= 4) - request_store (~> 1.0) loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -255,6 +250,8 @@ GEM minitest (>= 5.0, < 7) ruby-progressbar msgpack (1.8.0) + neighbor (1.2.0) + activerecord (>= 7.2) nenv (0.3.0) net-http (0.9.1) uri (>= 0.11.1) @@ -384,6 +381,10 @@ GEM activesupport (= 8.1.3) bundler (>= 1.15.0) railties (= 8.1.3) + rails-controller-testing (1.0.5) + actionpack (>= 5.0.1.rc1) + actionview (>= 5.0.1.rc1) + activesupport (>= 5.0.1.rc1) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest @@ -427,8 +428,6 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) - request_store (1.7.0) - rack (>= 1.4) responders (3.2.0) actionpack (>= 7.0) railties (>= 7.0) @@ -585,10 +584,10 @@ DEPENDENCIES jbuilder kamal listen (~> 3.5) - lograge mini_magick minitest-rails minitest-reporters + neighbor omniauth omniauth-rails_csrf_protection omniauth_openid_connect @@ -597,6 +596,7 @@ DEPENDENCIES propshaft puma (>= 5.0) rails (~> 8.1.0) + rails-controller-testing rails-erd rails-healthcheck rails_icons (~> 1.3) diff --git a/Makefile b/Makefile index b83b2e3..f2799a9 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ -include .env.production .env.development -REGISTRY := registry.coolify1.workaround.org +REGISTRY := git.workaround.org/chaas -stage: TAG=stage +stage: TAG=latest stage: build tag push curl -s -H "Authorization: Bearer $(COOLIFY_STAGE_DEPLOY_TOKEN)" \ -X POST "$(COOLIFY_STAGE_DEPLOY_WEBHOOK)" @@ -15,10 +15,11 @@ prod: build tag push -X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)" build: - podman build -t debshots-$(TAG) . + docker build --build-arg GIT_REV=$(shell git rev-parse --short HEAD) -t debshots-$(TAG) . tag: - podman tag debshots-$(TAG) $(REGISTRY)/debshots-web:$(TAG) + docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG) push: - podman push $(REGISTRY)/debshots-web:$(TAG) + docker push $(REGISTRY)/debshots:$(TAG) + diff --git a/README.md b/README.md index 55facc4..c330d9c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,42 @@ https://salsa.debian.org/debian/debshots/-/issues Check out the doc/README.Development.md +## Package search + +The package search combines several strategies, tried in this order: + +1. **Exact package name**: Searching for `vim` shows exactly this + package without calling any external service. +2. **Name prefix matches**: Packages whose name starts with the query + rank first, e.g. searching for `sqlite` promotes `sqlite3`, + `sqlitebrowser` and friends above all other results. +3. **Compound word splitting**: Packages whose name contains *all* + query tokens match too, so `sqlite browser` finds `sqlitebrowser`. +4. **Semantic search**: The query is embedded via the vector service + (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among + the package description embeddings are returned, using the pgvector + extension with an HNSW cosine index. Embeddings are computed from + the package description and long description and can be (re-)built + with `bin/rails debshots:compute_vectors`. +5. **Full-text fallback**: If the vector service cannot be reached, + classic PostgreSQL full-text search takes over. + +Queries that share no vocabulary with the package data (gibberish, +keyboard mash) return an honest empty result instead of random +semantic matches. + +**Why is this package in my results?** Each result knows which search +strategy produced it. The controller exposes this as +`@search_sources` (a hash mapping package ids to `:exact`, +`:name_prefix`, `:name_contains`, `:semantic` or `:fulltext`). For +debugging you can render a small badge on every result card by adding +`show_sources=1` to the URL, e.g. +`https://screenshots.debian.net/packages?search=sqlite&show_sources=1`. + +Package pages also show semantically **related packages**, computed as +a nearest neighbor query against the package's own description +embedding - no external service call involved. + ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index b242467..602595b 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -30,6 +30,9 @@ class PackagesController < ApplicationController @screenshots = @package.screenshots.accessible_by(current_ability, :view).paginate( page: @page, per_page: 6 ) + # Semantic recommendations based on the package's own description + # embedding - a plain nearest neighbor query, no service call. + @related_packages = @package.related_packages end end @@ -359,10 +362,15 @@ class PackagesController < ApplicationController def query_packages packages = Package # .order(visits: :desc) + # Maps package id to the search strategy that put it into the + # result list: :exact, :name_prefix, :name_contains, :semantic or + # :fulltext. Empty when browsing without a search term. + @search_sources = {} + # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" - packages = packages.general_search(params[:search]) + packages = search_packages(params[:search]) end case params[:show] @@ -378,6 +386,56 @@ class PackagesController < ApplicationController packages end + # Text search for packages: an exact package name match wins without + # touching the vector service. Otherwise packages whose name starts + # with the query (or contains all query tokens, catching compound + # words like "sqlite browser" -> "sqlitebrowser") are promoted above + # the semantic (vector) nearest neighbor results. Classic PostgreSQL + # full-text search kicks in when the vector service cannot be reached. + MAX_PROMOTED_NAME_MATCHES = 8 + MAX_SEMANTIC_RESULTS = 100 + private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS + + def search_packages(query) + exact_match = Package.find_by(name: query) + if exact_match + @search_sources[exact_match.id] = :exact + return Package.where(id: exact_match.id) + end + + prefix_matches = Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES).to_a + token_matches = Package.name_contains_all(query.to_s.split(/\s+/)).limit( + MAX_PROMOTED_NAME_MATCHES + ).to_a + prefix_matches.each { |package| @search_sources[package.id] ||= :name_prefix } + token_matches.each { |package| @search_sources[package.id] ||= :name_contains } + + semantic = [] + if Package.lexical_overlap?(query) + begin + semantic = Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a + semantic.each { |package| @search_sources[package.id] ||= :semantic } + rescue Vectorizer::Error => e + logger.error "Semantic search failed: #{e.message}" + flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' + semantic = Package.general_search(query).limit(100).to_a + semantic.each { |package| @search_sources[package.id] ||= :fulltext } + end + else + logger.debug "Query '#{query}' has no lexical overlap with package data" + end + + # Deduplicate by NAME, not id: if the database ever contains + # duplicate rows (see the packages_name_key incident) users should + # not see the same package twice. + results = (prefix_matches + token_matches + semantic).uniq(&:name) + + # Queries that share no vocabulary with our package data can produce + # no meaningful results. Skip the vector service round-trip and show + # an honest empty result instead of random packages. + results.empty? ? Package.none : results + end + # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index d7dec2c..b742a6b 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -1,4 +1,22 @@ module PackagesHelper + # Search result origin badges are only shown when explicitly + # requested via ?show_sources=1 + def search_sources_visible? + params[:show_sources].present? + end + + # Human readable label for the search strategy that put a package + # into the search results (see PackagesController#search_packages) + def search_source_label(source) + { + exact: 'exact match', + name_prefix: 'name match', + name_contains: 'name match', + semantic: 'similar', + fulltext: 'full-text' + }[source] + end + # Return the description and/or the version of the package def screenshot_caption(screenshot) str = [] diff --git a/app/models/package.rb b/app/models/package.rb index 441f5c4..c2e31de 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open() require 'json' require 'deb_importer' +# Search and embedding logic keeps accumulating here; the class is +# cohesive enough that splitting it is not worth the indirection. +# rubocop:disable Metrics/ClassLength class Package < ApplicationRecord # PostgreSQL-based full-text search: # https://github.com/Casecommons/pg_search @@ -26,6 +29,11 @@ class Package < ApplicationRecord inverse_of: :package, dependent: :destroy + # Nearest neighbor search using pgvector. The "embedding" column stores + # a vector computed from description + long_description by an external + # web service (see Vectorizer and lib/tasks/vectorize_packages.rake). + has_neighbors :embedding + # default_scope { # order('name ASC') # } @@ -40,6 +48,103 @@ class Package < ApplicationRecord long_description.split(/\n\.\n/).first if long_description end + # The text that the embedding vector is computed from: description + # and long description combined. + def embedding_text + [description, long_description].compact.join("\n") + end + + # Compute and store the embedding vector for this package by asking + # the external vector service. Raises Vectorizer::Error on failure. + # Returns true if a vector was saved. + def update_embedding! + text = embedding_text + return false if text.blank? + + update_column(:embedding, Vectorizer.embed(text)) + end + + # Return a relation of packages whose stored embeddings are closest to + # the given query vector (an array of floats). Records get a + # "neighbor_distance" attribute added (cosine distance, lower is better). + # The returned relation can still be chained (e.g. filtered or paginated). + # + # Only returns packages that already have an embedding computed. + def self.nearest_to_vector(vector, limit: nil) + neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine') + limit ? neighbors.limit(limit) : neighbors + end + + # Turn a free-text search string into a vector via the external web + # service and find the closest packages. + def self.nearest_to_text(text, limit: nil) + return none if text.blank? + + nearest_to_vector(Vectorizer.embed(text), limit: limit) + end + + # Packages whose name starts with the given string - used to promote + # obvious matches like "sqlite3" for a "sqlite" search. + def self.name_starts_with(text) + return none if text.blank? + + where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name) + end + + # Packages whose name contains all given tokens as substrings. Catches + # compound words split by the user, e.g. "sqlite browser" finding the + # "sqlitebrowser" package. + def self.name_contains_all(tokens) + tokens = tokens.select { |token| token.length >= 2 } + return none if tokens.empty? + + scope = self + tokens.each do |token| + scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\') + end + scope.order(:name) + end + + def self.escape_for_like(text) + # Escape LIKE wildcards so that package name characters like "+" or "." + # cannot break the pattern + text.gsub(/[\\%_]/) { |char| "\\#{char}" } + end + + # Packages semantically similar to this one, based on the stored + # description embedding. Returns an empty relation for packages that + # have no embedding (yet). + def related_packages(limit: 6) + return Package.none unless embedding + + Package.nearest_to_vector(embedding, limit: limit + 1) + .where.not(id: id).limit(limit) + end + + # Check whether the given text shares at least one lexeme with any + # package name/description in the database. Queries without such an + # overlap (gibberish, keyboard mash, unsupported languages) can + # neither match full-text nor produce meaningful semantic results - + # the embedding model maps unknown tokens close to the corpus + # centroid, making them look like good matches. + def self.lexical_overlap?(text) + tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) } + return false if tokens.empty? + + # Pin the same text search dictionary that the packages_fts index + # uses - the server-wide default can be anything. + tsquery = tokens.map do |token| + sanitize_sql(["plainto_tsquery('english', ?)", token]) + end.join(' || ') + # The expression must match the packages_fts GIN index definition + # so that Postgres can use the index. + where( + "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \ + "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \ + "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})" + ).exists? + end + # Return a query of all packages that have screenshots def self.with_screenshots # Query for all packages who's ID appears in a screenshot's "package_id" field @@ -113,3 +218,4 @@ class Package < ApplicationRecord end end end +# rubocop:enable Metrics/ClassLength diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 84d8599..b8e09c9 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -14,7 +14,7 @@ - if @packages.any? - if @view_style==:grid .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium" - - @packages.all.each do |pkg| + - @packages.each do |pkg| .cell.pkgcard data-equalizer-watch=true a.black href=package_path(name: pkg.name) .image @@ -24,6 +24,9 @@ = pkg.name .text = pkg.description + - if search_sources_visible? && @search_sources[pkg.id] + .text.search-source + span.label.secondary = search_source_label(@search_sources[pkg.id]) - elsif @view_style==:list - @packages.to_a.each do |pkg| .grid-x.grid-margin-x.listview @@ -37,6 +40,9 @@ a href=package_path(name: pkg.name) =pkg.name p =pkg.description + - if search_sources_visible? && @search_sources[pkg.id] + p + span.label.secondary = search_source_label(@search_sources[pkg.id]) .listview.longdescription = pkg.long_description_first_paragraph diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index 03c9609..7443302 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -70,6 +70,19 @@ / #metadata is the anchor for this sticky container. = render(partial: 'details_rightbox', locals: {pkg: @package}) + / Semantically similar packages, based on the package's description embedding + - if @related_packages.any? + #related + h3 Related packages + .grid-x.grid-margin-x.medium-up-2.large-up-3 data-equalizer=true + - @related_packages.each do |pkg| + .cell.pkgcard data-equalizer-watch=true + a.black href=package_path(name: pkg.name) + .text.pkgname + = pkg.name + .text.small + = pkg.description + javascript: // Where to send POST requests for uploads let ajax_upload_url = '#{{upload_receive_json_path}}'; diff --git a/config/environments/production.rb b/config/environments/production.rb index ced9b3c..b1c0f32 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -37,15 +37,6 @@ Rails.application.configure do config.log_tags = [:request_id] config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) - # Lograge - structured request logging - config.lograge.enabled = true - config.lograge.custom_options = lambda do |event| - { - request_id: event.payload[:request_id], - user_id: event.payload[:user_id] - } - end - # Change to "debug" to log everything (including potentially personally-identifiable information!). config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info') diff --git a/config/initializers/will_paginate.rb b/config/initializers/will_paginate.rb new file mode 100644 index 0000000..e36b7b3 --- /dev/null +++ b/config/initializers/will_paginate.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Enable will_paginate's Array#paginate so that merged search results +# (name prefix matches + semantic results) can be paginated like +# ActiveRecord relations. +require 'will_paginate/array' diff --git a/db/migrate/20260821203833_install_neighbor_vector.rb b/db/migrate/20260821203833_install_neighbor_vector.rb new file mode 100644 index 0000000..d591f1c --- /dev/null +++ b/db/migrate/20260821203833_install_neighbor_vector.rb @@ -0,0 +1,5 @@ +class InstallNeighborVector < ActiveRecord::Migration[8.1] + def change + enable_extension "vector" + end +end diff --git a/db/migrate/20260821210000_add_embedding_to_packages.rb b/db/migrate/20260821210000_add_embedding_to_packages.rb new file mode 100644 index 0000000..62825f9 --- /dev/null +++ b/db/migrate/20260821210000_add_embedding_to_packages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Adds a vector column to store text embeddings of the package +# description and long description. The vectors are computed by an +# external web service and used for nearest neighbor search. +# +# The pgvector extension is enabled by the InstallNeighborVector migration. +class AddEmbeddingToPackages < ActiveRecord::Migration[8.1] + def change + add_column :packages, :embedding, :vector, limit: 384 + + # HNSW index for fast approximate nearest neighbor searches + add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops + end +end diff --git a/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb b/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb new file mode 100644 index 0000000..89efc03 --- /dev/null +++ b/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Packages must be unique by name - the same package is listed once per +# architecture/component in the Debian archive and the importer looks it up +# by name before inserting, so anything that lets two rows for one name appear +# (e.g. concurrent import runs) breaks the site. Production had accidentally +# relied on a constraint that was only ever added ad-hoc on some databases +# and never tracked as a migration; this makes db:migrate the single source +# of truth. +class AddUniqueNameConstraintToPackages < ActiveRecord::Migration[8.1] + def up + add_unique_constraint :packages, :name, name: :packages_name_key unless constraint_exists? + end + + def down + remove_unique_constraint :packages, :name, name: :packages_name_key if constraint_exists? + end + + private + + def constraint_exists? + select_value(<<~SQL.squish) + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'packages'::regclass AND conname = 'packages_name_key' AND contype = 'u' + SQL + end +end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index 2c57b69..433a8e2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,133 +10,131 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do - # These are extensions that must be enabled to support this database - enable_extension 'pg_catalog.plpgsql' +ActiveRecord::Schema[8.1].define(version: 2026_09_04_120000) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + enable_extension "vector" - create_table 'action_mailbox_inbound_emails', force: :cascade do |t| - t.integer 'status', default: 0, null: false - t.string 'message_id', null: false - t.string 'message_checksum', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['message_id', 'message_checksum'], -name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true + create_table "action_mailbox_inbound_emails", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "message_checksum", null: false + t.string "message_id", null: false + t.integer "status", default: 0, null: false + t.datetime "updated_at", null: false + t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true end - create_table 'action_text_rich_texts', force: :cascade do |t| - t.string 'name', null: false - t.text 'body' - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', -unique: true + create_table "action_text_rich_texts", force: :cascade do |t| + t.text "body" + t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.datetime "updated_at", null: false + t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true end - create_table 'active_storage_attachments', force: :cascade do |t| - t.string 'name', null: false - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.bigint 'blob_id', null: false - t.datetime 'created_at', precision: nil, null: false - t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id' - t.index ['record_type', 'record_id', 'name', 'blob_id'], -name: 'index_active_storage_attachments_uniqueness', unique: true + create_table "active_storage_attachments", force: :cascade do |t| + t.bigint "blob_id", null: false + t.datetime "created_at", precision: nil, null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end - create_table 'active_storage_blobs', force: :cascade do |t| - t.string 'key', null: false - t.string 'filename', null: false - t.string 'content_type' - t.text 'metadata' - t.bigint 'byte_size', null: false - t.string 'checksum' - t.datetime 'created_at', precision: nil, null: false - t.string 'service_name', null: false - t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true + create_table "active_storage_blobs", force: :cascade do |t| + t.bigint "byte_size", null: false + t.string "checksum" + t.string "content_type" + t.datetime "created_at", precision: nil, null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" + t.string "service_name", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end - create_table 'active_storage_variant_records', force: :cascade do |t| - t.bigint 'blob_id', null: false - t.string 'variation_digest', null: false - t.index ['blob_id', 'variation_digest'], -name: 'index_active_storage_variant_records_uniqueness', unique: true + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end - create_table 'logs', id: :serial, force: :cascade do |t| - t.string 'message' - t.string 'level' - t.string 'section' - t.inet 'ip_address' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.integer 'user_id' - t.integer 'package_id' - t.integer 'screenshot_id' + create_table "logs", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.inet "ip_address" + t.string "level" + t.string "message" + t.integer "package_id" + t.integer "screenshot_id" + t.string "section" + t.datetime "updated_at", precision: nil, null: false + t.integer "user_id" end - create_table 'packages', id: :serial, force: :cascade do |t| - t.string 'name', null: false - t.string 'description', limit: 80 - t.string 'section', limit: 50 - t.string 'maintainer', limit: 100 - t.string 'maintainer_email', limit: 100 - t.string 'homepage', limit: 400 - t.string 'version', limit: 200 - t.text 'long_description' - t.string 'origin', limit: 80 - t.datetime 'created_at', precision: nil - t.datetime 'updated_at', precision: nil - t.integer 'visits', default: 0 - t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", -name: 'packages_fts', using: :gin - t.unique_constraint ['name'], name: 'packages_name_key' + create_table "packages", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil + t.string "description", limit: 80 + t.vector "embedding", limit: 384 + t.string "homepage", limit: 400 + t.text "long_description" + t.string "maintainer", limit: 100 + t.string "maintainer_email", limit: 100 + t.string "name", null: false + t.string "origin", limit: 80 + t.string "section", limit: 50 + t.datetime "updated_at", precision: nil + t.string "version", limit: 200 + t.integer "visits", default: 0 + t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin + t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw + t.unique_constraint ["name"], name: "packages_name_key" end - create_table 'screenshots', id: :serial, force: :cascade do |t| - t.integer 'package_id' - t.string 'version', limit: 50 - t.datetime 'created_at', precision: nil - t.string 'uploaderhash', limit: 72 - t.boolean 'approved', default: false, null: false - t.text 'description' - t.datetime 'updated_at', precision: nil - t.string 'image_fingerprint' - t.integer 'user_id', default: 0 - t.text 'simage_data' - t.inet 'uploaderip' - t.boolean 'hidden', default: false - t.index ['id', 'approved'], name: 'id_approved' - t.index ['id', 'uploaderhash'], name: 'id_uploaderhash' + create_table "screenshots", id: :serial, force: :cascade do |t| + t.boolean "approved", default: false, null: false + t.datetime "created_at", precision: nil + t.text "description" + t.boolean "hidden", default: false + t.string "image_fingerprint" + t.integer "package_id" + t.text "simage_data" + t.datetime "updated_at", precision: nil + t.string "uploaderhash", limit: 72 + t.inet "uploaderip" + t.integer "user_id", default: 0 + t.string "version", limit: 50 + t.index ["id", "approved"], name: "id_approved" + t.index ["id", "uploaderhash"], name: "id_uploaderhash" end - create_table 'users', id: :serial, force: :cascade do |t| - t.text 'name' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.string 'email', default: '', null: false - t.string 'encrypted_password', default: '', null: false - t.integer 'sign_in_count', default: 0, null: false - t.datetime 'current_sign_in_at', precision: nil - t.datetime 'last_sign_in_at', precision: nil - t.inet 'current_sign_in_ip' - t.inet 'last_sign_in_ip' - t.integer 'failed_attempts', default: 0, null: false - t.string 'unlock_token' - t.datetime 'locked_at', precision: nil - t.string 'provider' - t.string 'uid' - t.boolean 'admin_role', default: false - t.boolean 'moderator_role', default: false - t.boolean 'pseudo', default: false - t.integer 'approved_screenshots', default: 0 - t.integer 'rejected_screenshots', default: 0 - t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true + create_table "users", id: :serial, force: :cascade do |t| + t.boolean "admin_role", default: false + t.integer "approved_screenshots", default: 0 + t.datetime "created_at", precision: nil, null: false + t.datetime "current_sign_in_at", precision: nil + t.inet "current_sign_in_ip" + t.string "email", default: "", null: false + t.string "encrypted_password", default: "", null: false + t.integer "failed_attempts", default: 0, null: false + t.datetime "last_sign_in_at", precision: nil + t.inet "last_sign_in_ip" + t.datetime "locked_at", precision: nil + t.boolean "moderator_role", default: false + t.text "name" + t.string "provider" + t.boolean "pseudo", default: false + t.integer "rejected_screenshots", default: 0 + t.integer "sign_in_count", default: 0, null: false + t.string "uid" + t.string "unlock_token" + t.datetime "updated_at", precision: nil, null: false + t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true end - add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey' + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey" end diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 8f84233..5569c07 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -29,11 +29,47 @@ BLACKLIST_SECTION_PATTERN = [ %r{/?oldlibs$}, %r{/?libdevel$}, %r{/?cli-mono$}, + # Libraries never make useful screenshots - and unlike a "lib*" name + # pattern this does not hit applications like libreoffice (editors) + # or librecad (graphics) + %r{/?libs$}, +] + +# List of regular expressions. If the package's short description +# matches any of these then the package will not be imported. +# Transitional/dummy stub packages exist only to ease upgrades to a +# renamed or split package and never get a useful screenshot. Debian +# has no dedicated metadata field for this - the Developer's Reference +# (6.9.7) documents these exact phrases as the convention package +# maintainers use, and as what deborphan --guess-dummy looks for. +# Deliberately not matching a bare "dummy": too many real packages use +# that word for actual functionality (e.g. xserver-xorg-video-dummy, +# fence-agents-dummy, "double dummy solver" bridge card libraries). +BLACKLIST_DESCRIPTION_PATTERN = [ + /transitional/i, + /dummy package/i, + /dependency package/i, ] # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true +# Postgres advisory lock used to serialize update_from_deb_repos runs. +# Without this, a scheduled and a manual run (or two scheduled runs) can +# overlap: both do find_by+insert for the same name, and the second one +# either violates packages_name_key (aborting the whole import) or - on +# instances that never had that constraint - silently creates a duplicate. +IMPORT_ADVISORY_LOCK_KEY = 0x0DE65507 + +# Whether to delete packages that were not found in any configured +# source during a full import run (e.g. packages Debian has dropped +# entirely, or ones from a suite/architecture no longer configured). +# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking +# "orphaned" can also mean a mirror had a transient fetch problem, so +# update_from_deb_repos refuses to act on it unless every configured +# component/architecture was actually fetched successfully this run. +REMOVE_ORPHANED_PACKAGES = false + namespace :debshots do desc "Import/update package database from configured DEB repositories" @@ -43,11 +79,23 @@ namespace :debshots do stats_updated = 0 stats_removed = 0 + # Every package name encountered in any configured source this run, + # used to find orphans afterwards. A Hash is used as a cheap set. + seen_package_names = {} + # Set to true if any component/architecture could not be fetched, + # so we know not to trust seen_package_names for orphan removal. + sources_incomplete = false + repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO #Rails.logger.level = Logger::DEBUG + unless import_lock_acquired? + Rails.logger.warn 'Another import run already in progress (advisory lock). Skipping.' + exit 0 + end + Rails.logger.info "Importing Debian package information" repositories.each do |repository| @@ -74,6 +122,7 @@ namespace :debshots do packages = release.packages(component, architecture) unless packages Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" + sources_incomplete = true next end @@ -82,8 +131,11 @@ namespace :debshots do packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" + seen_package_names[package[:Package]] = true - if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] + if package_name_blacklisted?(package[:Package]) or + package_section_blacklisted?(package[:Section]) or + package_description_blacklisted?(package[:Description]) # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] @@ -97,24 +149,37 @@ namespace :debshots do next end # if blacklisted - db_package = Package.find_by name: package[:Package] - if db_package # package exists in the database - # check if the parsed package data has a newer version - current_version = DebImporter::Version.new(db_package.version) - new_version = DebImporter::Version.new(package[:Version]) + begin + db_package = Package.find_by name: package[:Package] + if db_package # package exists in the database + # check if the parsed package data has a newer version + current_version = DebImporter::Version.new(db_package.version) + new_version = DebImporter::Version.new(package[:Version]) - Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" - if new_version > current_version - # update the package information from new data + Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" + if new_version > current_version + # update the package information from new data + update_data(package, db_package) + Rails.logger.info "Updating package information" + stats_updated += 1 + end + else # new package - create it in the database + Rails.logger.debug "No such package in our database. Creating one." + db_package = Package.new + update_data(package, db_package) + stats_added += 1 + end + rescue ActiveRecord::RecordNotUnique + # Lost a create-race: another process inserted this package name + # between our find and our insert (a web upload, or an import on + # an instance without the advisory lock). Treat it as found and + # update instead of aborting the whole run. + Rails.logger.warn "Package '#{package[:Package]}' was created " \ + 'concurrently. Updating it instead.' + if (db_package = Package.find_by(name: package[:Package])) update_data(package, db_package) - Rails.logger.info "Updating package information" stats_updated += 1 end - else # new package - create it in the database - Rails.logger.debug "No such package in our database. Creating one." - db_package = Package.new - update_data(package, db_package) - stats_added += 1 end # unless db_package @@ -138,6 +203,9 @@ namespace :debshots do Rails.logger.info "--------------------------------------------" end # repositories.each + stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete) + Rails.logger.info "#{stats_removed} packages removed in total" + end # task desc 'Update long description from i18n file' @@ -276,3 +344,80 @@ def package_section_blacklisted?(section) end return false end + +# Check if a package's short description marks it as a transitional or +# dummy stub package (e.g. "transitional package", "transitional dummy +# package for foo"). These never get a useful screenshot. +def package_description_blacklisted?(description) + BLACKLIST_DESCRIPTION_PATTERN.each do |pattern| + if description.to_s =~ pattern + Rails.logger.debug " > Blacklisted by description ('#{description}' matches '#{pattern}')" + return true + end + end + false +end + +# Remove packages that were not encountered in any configured source +# during this import run. Refuses to act if any component/architecture +# failed to fetch (sources_incomplete) or if seen_names came back empty +# (both would otherwise risk wiping out packages that are still very +# much alive in Debian, just missed because of a network hiccup). +# Returns the number of packages actually removed. +def orphan_removal_blocked?(seen_names, sources_incomplete) + if sources_incomplete + Rails.logger.warn ' > Skipping: not all configured components/architectures could be ' \ + 'fetched this run. Re-run when all sources are reachable.' + return true + end + + if seen_names.empty? + Rails.logger.warn ' > Skipping: no packages were seen this run at all - refusing to ' \ + 'treat that as "everything is orphaned".' + return true + end + + false +end + +def find_orphaned_package_ids(seen_names) + Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids| + ids << id unless seen_names.key?(name) + end +end + +def destroy_orphaned_packages(ids) + removed = 0 + Package.where(id: ids).find_each do |package| + Rails.logger.info " > Removing orphaned package '#{package.name}' from database" + package.destroy + removed += 1 + end + removed +end + +def remove_orphaned_packages(seen_names, sources_incomplete) + Rails.logger.info 'Checking for packages no longer found in any configured source' + return 0 if orphan_removal_blocked?(seen_names, sources_incomplete) + + orphan_ids = find_orphaned_package_ids(seen_names) + + unless REMOVE_ORPHANED_PACKAGES + Rails.logger.info " > #{orphan_ids.size} packages are orphaned (dry run - set " \ + 'REMOVE_ORPHANED_PACKAGES to actually remove them)' + return 0 + end + + destroy_orphaned_packages(orphan_ids) +end + +# Try to take the advisory lock guarding update_from_deb_repos. The lock lives +# on this database connection/session, so it is released automatically when the +# process exits - no explicit unlock needed. Returns true if this run is the +# one that may proceed. +def import_lock_acquired? + result = ActiveRecord::Base.connection.execute( + "SELECT pg_try_advisory_lock(#{IMPORT_ADVISORY_LOCK_KEY})" + ) + result.first['pg_try_advisory_lock'] == true +end diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake new file mode 100644 index 0000000..aee5d45 --- /dev/null +++ b/lib/tasks/vectorize_packages.rake @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +# Long rake task bodies are normal; the block-length cop is meant for +# application code. +# rubocop:disable Metrics/BlockLength +namespace :debshots do + desc 'Compute embedding vectors for all packages based on their description ' \ + 'and long description (missing ones only unless FORCE=1)' + task compute_vectors: :environment do + logger = Logger.new($stdout) + logger.level = Logger::INFO + + force = ENV['FORCE'].present? + # Packages without any description text cannot be embedded at all - + # exclude them so they do not fail on every single run. + base_scope = Package.where("COALESCE(description, '') <> '' OR " \ + "COALESCE(long_description, '') <> ''") + scope = force ? base_scope : base_scope.where(embedding: nil) + total = scope.count + + if total.zero? + logger.info 'No packages need vector computation.' + next + end + + logger.info "Computing vectors for #{total} packages " \ + "(#{force ? 'recomputing' : 'missing only'}, " \ + "service: #{Vectorizer.service_url})" + + done = 0 + failed = 0 + + # Remember the most recent SQL statement so failures can be traced + # to the exact query that triggered them. + last_sql = nil + ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| + last_sql = args.last[:sql] + end + + scope.find_each do |package| + begin + raise ArgumentError, 'no description text' if package.embedding_text.blank? + + package.update_embedding! + done += 1 + rescue StandardError => e + # A single broken package must never stop the whole run - log + # enough detail to diagnose it and move on. + failed += 1 + logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}" + logger.error "Last SQL: #{last_sql}" + logger.error e.backtrace.first(15).join("\n") if e.backtrace + end + + processed = done + failed + next unless (processed % 100).zero? || processed == total + + percent = (processed.to_f / total * 100).round(1) + logger.info "Progress: #{processed}/#{total} (#{percent}%)" + end + + logger.info "Done. #{done} vectors computed, #{failed} failures." + end +end +# rubocop:enable Metrics/BlockLength diff --git a/lib/vectorizer.rb b/lib/vectorizer.rb new file mode 100644 index 0000000..84df3c4 --- /dev/null +++ b/lib/vectorizer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' + +# Client for the external web service that turns text into embedding +# vectors for nearest neighbor search (pgvector). +# +# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...] +# +# The service URL can be overridden with the VECTOR_SERVICE_URL +# environment variable. +class Vectorizer + # Raised when the vector service cannot be reached or returns garbage + class Error < StandardError; end + + DEFAULT_URL = 'https://vector-search.workaround.org/vector' + + class << self + # Compute the embedding vector for the given text. + # Returns an array of floats. + def embed(text) + new.embed(text) + end + + def service_url + ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL) + end + end + + def embed(text) + response = post_text(text) + + unless response.is_a?(Net::HTTPSuccess) + raise Error, "Vector service returned HTTP #{response.code}" + end + + JSON.parse(response.body).fetch('vector') + rescue JSON::ParserError => e + raise Error, "Vector service returned invalid JSON: #{e.message}" + rescue KeyError + raise Error, 'Vector service response did not contain a "vector" field' + end + + private + + def post_text(text) + uri = URI.parse(self.class.service_url) + request = build_request(uri, text) + + options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 } + Net::HTTP.start(uri.host, uri.port, **options) do |http| + http.request(request) + end + rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e + raise Error, "Could not reach vector service: #{e.message}" + end + + def build_request(uri, text) + request = Net::HTTP::Post.new(uri.request_uri) + request['Content-Type'] = 'application/json' + request.body = { text: text }.to_json + + # Optional Bearer token if the service runs with API_SECRET set + secret = ENV['VECTOR_SERVICE_SECRET'] + request['Authorization'] = "Bearer #{secret}" if secret.present? + + request + end +end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index c9eae31..d72ad95 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class PackagesControllerTest < ActionController::TestCase @@ -20,6 +22,118 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'A program to browse web sites.' end + test 'exact package name search wins without calling the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'vim', show_sources: '1' } + assert_response :success + assert_select 'div', 'vim' + assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources)) + assert_select '.search-source .label', 'exact match' + end + end + + test 'gibberish search shows no results and never calls the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'asdfgh zzzz' } + assert_response :success + # the grid renders every package in the database when no search + # filter applies - so an empty result must not fall back to that + assert_select 'div', text: /package-\d+/, count: 0 + assert_empty assigns(:search_sources) + end + end + + test 'name prefix matches rank above semantic results' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do + get :grid, params: { search: 'package' } + assert_response :success + # fixtures create package-0..package-99 - prefix hits must render + assert_select 'div', 'package-0' + # ...and before any semantically found package + assert response.body.index('package-0') < response.body.index('vim') + sources = assigns(:search_sources) + assert_equal :name_prefix, sources[Package.find_by(name: 'package-0').id] + assert_equal :semantic, sources[packages(:vim).id] + end + end + + test 'prefix matches and semantic results are deduplicated' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do + get :grid, params: { search: 'fire' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + # the higher tier wins the attribution + assert_equal :name_prefix, assigns(:search_sources)[packages(:firefox).id] + end + end + + test 'compound words in search match packages with joined names' do + # "fire fox" should find "firefox" via name substring matching + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.none }) do + get :grid, params: { search: 'fire fox', show_sources: '1' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + assert_equal({ packages(:firefox).id => :name_contains }, + assigns(:search_sources)) + assert_select '.search-source .label', 'name match' + end + end + + test 'search source badges stay hidden without show_sources param' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.none }) do + get :grid, params: { search: 'fire fox' } + assert_response :success + assert_select '.search-source', count: 0 + end + end + + test 'details shows semantically related packages' do + packages(:vim).update_column(:embedding, Array.new(384, 0.25)) + packages(:firefox).update_column(:embedding, Array.new(384, 0.75)) + + get :details, params: { name: 'vim' } + assert_response :success + assert_select '#related .pkgname', text: 'firefox' + end + + test 'details without embedding renders no related section' do + get :details, params: { name: 'vim' } + assert_response :success + assert_select '#related', count: 0 + end + + test 'search uses semantic (vector) results' do + relation = Package.where(name: packages(:vim).name) + with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + assert_equal({ packages(:vim).id => :semantic }, assigns(:search_sources)) + end + end + + # There is deliberately no fallback for an empty vector search + # result: KNN always returns something as long as packages have + # embeddings. Full-text search is only used when the vector service + # cannot be reached. + + test 'search falls back to full-text search when vector service fails' do + with_stubbed_method(Vectorizer, :embed, + ->(_text) { raise Vectorizer::Error, 'service down' }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + assert flash[:error].present? + assert_equal({ packages(:vim).id => :fulltext }, assigns(:search_sources)) + end + end + # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') diff --git a/test/models/package_test.rb b/test/models/package_test.rb index 124f6c4..36562a2 100644 --- a/test/models/package_test.rb +++ b/test/models/package_test.rb @@ -1,7 +1,107 @@ +# frozen_string_literal: true + require 'test_helper' class PackageTest < ActiveSupport::TestCase - # test "the truth" do - # assert true - # end + test 'embedding_text combines description and long description' do + package = packages(:firefox) + assert_equal "A web browser\nA program to browse web sites.", package.embedding_text + end + + test 'embedding_text works with missing long description' do + package = packages(:firefox) + package.long_description = nil + assert_equal 'A web browser', package.embedding_text + end + + test 'update_embedding! stores the vector returned by the vector service' do + vector = Array.new(384) { |i| i / 384.0 } + with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do + package = packages(:vim) + assert package.update_embedding! + # The vector type stores float32 so allow small rounding differences + package.reload.embedding.to_a.each_with_index do |actual, i| + assert_in_delta vector[i], actual, 1e-6 + end + end + end + + test 'update_embedding! does nothing without description text' do + package = packages(:firefox) + package.description = nil + package.long_description = nil + with_stubbed_method(Vectorizer, :embed, + ->(_text) { flunk 'should not call the vector service' }) do + assert_not package.update_embedding! + end + end + + test 'nearest_to_text returns none for blank text' do + assert_empty Package.nearest_to_text('') + assert_empty Package.nearest_to_text(nil) + end + + test 'name_starts_with matches name prefixes case-insensitively' do + assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name + assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name + assert_empty Package.name_starts_with('') + assert_empty Package.name_starts_with(nil) + end + + test 'name_contains_all finds compound words split by spaces' do + assert_includes Package.name_contains_all(%w[fire fox]).map(&:name), + packages(:firefox).name + # all tokens must be present in the name + assert_empty Package.name_contains_all(%w[firefox zzzz]) + assert_empty Package.name_contains_all([]) + # single character tokens would flood results - ignore them + assert_empty Package.name_contains_all(['f']) + end + + test 'lexical_overlap? pins the english dictionary regardless of server config' do + # Regression: with default_text_search_config = german (as on some + # servers) 'browser' stems to 'brows' which never matches the + # english-stemmed index. + connection = ActiveRecord::Base.connection + original = connection.select_value('SHOW default_text_search_config') + connection.execute("SET default_text_search_config = 'german'") + assert Package.lexical_overlap?('browser') + ensure + connection&.execute("SET default_text_search_config = '#{original}'") + end + + test 'lexical_overlap? detects real words vs gibberish' do + # words present in fixture package descriptions + assert Package.lexical_overlap?('browser') + assert Package.lexical_overlap?('email client for vim syntax') + assert Package.lexical_overlap?('the browser') # stopwords are ignored + # gibberish, numbers and empty strings share no vocabulary + assert_not Package.lexical_overlap?('asdfgh zzzz') + assert_not Package.lexical_overlap?('123 !!!') + assert_not Package.lexical_overlap?('the') # stopword-only query + assert_not Package.lexical_overlap?('') + assert_not Package.lexical_overlap?(nil) + end + + test 'nearest_to_text embeds the query and searches' do + vector = Array.new(384, 0.25) + embed_called_with = nil + with_stubbed_method(Vectorizer, :embed, + lambda { |text| + embed_called_with = text + vector + }) do + received = [] + results = with_stubbed_method(Package, :nearest_neighbors, + lambda { |_field, received_vector, **_opts| + received << received_vector + [packages(:vim)] + }) do + Package.nearest_to_text('some search text') + end + assert_equal [packages(:vim)], results + assert_equal 'some search text', embed_called_with + assert_equal [vector], received + end + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 4889a19..1d786a4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) require 'rails/test_help' @@ -15,6 +17,23 @@ module ActiveSupport class TestCase # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. fixtures :all + + # Replace a class-level method with a fixed return value or lambda + # for the duration of the block. Used to keep tests away from + # external services like the vector embedding web service. + def with_stubbed_method(klass, method_name, value_or_lambda) + original = klass.method(method_name) + klass.define_singleton_method(method_name) do |*args, **kwargs| + if value_or_lambda.respond_to?(:call) + value_or_lambda.call(*args, **kwargs) + else + value_or_lambda + end + end + yield + ensure + klass.define_singleton_method(method_name, original) + end end end