From 4948fe50e358d3991a1b3fa328f78001894439a4 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sat, 22 Aug 2026 22:07:56 +0200 Subject: [PATCH 01/22] Add semantic package search via pgvector embeddings Package text search now combines several strategies: exact name match, name prefix promotion, compound word splitting ("sqlite browser" finds "sqlitebrowser") and semantic nearest neighbor search over description embeddings computed by an external embedding service (all-MiniLM-L6-v2, 384 dims) stored with the pgvector extension. Classic PostgreSQL full-text search remains as fallback when the vector service is unreachable. Gibberish queries without lexical overlap with the package data return empty results instead of random matches. Embeddings can be backfilled with bin/rails debshots:compute_vectors. Also drops the unused lograge gem from the Gemfile. --- Gemfile | 7 +- Gemfile.lock | 11 +- README.md | 24 ++ app/controllers/packages_controller.rb | 47 +++- app/models/package.rb | 96 ++++++++ app/views/packages/browse.slim | 2 +- config/initializers/will_paginate.rb | 6 + .../20260821203833_install_neighbor_vector.rb | 5 + ...0260821210000_add_embedding_to_packages.rb | 15 ++ db/schema.rb | 218 +++++++++--------- lib/tasks/vectorize_packages.rake | 84 +++++++ lib/vectorizer.rb | 70 ++++++ test/controllers/packages_controller_test.rb | 77 +++++++ test/models/package_test.rb | 106 ++++++++- test/test_helper.rb | 19 ++ 15 files changed, 662 insertions(+), 125 deletions(-) create mode 100644 config/initializers/will_paginate.rb create mode 100644 db/migrate/20260821203833_install_neighbor_vector.rb create mode 100644 db/migrate/20260821210000_add_embedding_to_packages.rb create mode 100644 lib/tasks/vectorize_packages.rake create mode 100644 lib/vectorizer.rb diff --git a/Gemfile b/Gemfile index c1e1ea4..6e1b32d 100644 --- a/Gemfile +++ b/Gemfile @@ -41,8 +41,7 @@ gem 'mini_magick' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false -# Structured logging for production -gem 'lograge' + # Deploy this application anywhere as a Docker container [https://kamal-deploy.org] gem 'kamal', require: false @@ -124,6 +123,10 @@ gem 'will_paginate-foundation' # Full-text search in PostgreSQL gem 'pg_search' +# Nearest neighbor search for PostgreSQL using the pgvector extension +# https://github.com/ankane/neighbor +gem 'neighbor' + # Use SLIM as our templating language gem 'slim-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 202b1e2..9cf75ac 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -224,11 +224,6 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.7.0) - lograge (0.14.0) - actionpack (>= 4) - activesupport (>= 4) - railties (>= 4) - request_store (~> 1.0) loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -255,6 +250,8 @@ GEM minitest (>= 5.0, < 7) ruby-progressbar msgpack (1.8.0) + neighbor (1.2.0) + activerecord (>= 7.2) nenv (0.3.0) net-http (0.9.1) uri (>= 0.11.1) @@ -427,8 +424,6 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) - request_store (1.7.0) - rack (>= 1.4) responders (3.2.0) actionpack (>= 7.0) railties (>= 7.0) @@ -585,10 +580,10 @@ DEPENDENCIES jbuilder kamal listen (~> 3.5) - lograge mini_magick minitest-rails minitest-reporters + neighbor omniauth omniauth-rails_csrf_protection omniauth_openid_connect diff --git a/README.md b/README.md index 55facc4..deaef3a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,30 @@ https://salsa.debian.org/debian/debshots/-/issues Check out the doc/README.Development.md +## Package search + +The package search combines several strategies, tried in this order: + +1. **Exact package name**: Searching for `vim` shows exactly this + package without calling any external service. +2. **Name prefix matches**: Packages whose name starts with the query + rank first, e.g. searching for `sqlite` promotes `sqlite3`, + `sqlitebrowser` and friends above all other results. +3. **Compound word splitting**: Packages whose name contains *all* + query tokens match too, so `sqlite browser` finds `sqlitebrowser`. +4. **Semantic search**: The query is embedded via the vector service + (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among + the package description embeddings are returned, using the pgvector + extension with an HNSW cosine index. Embeddings are computed from + the package description and long description and can be (re-)built + with `bin/rails debshots:compute_vectors`. +5. **Full-text fallback**: If the vector service cannot be reached, + classic PostgreSQL full-text search takes over. + +Queries that share no vocabulary with the package data (gibberish, +keyboard mash) return an honest empty result instead of random +semantic matches. + ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index b242467..726f4f0 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -362,7 +362,7 @@ class PackagesController < ApplicationController # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" - packages = packages.general_search(params[:search]) + packages = search_packages(params[:search]) end case params[:show] @@ -378,6 +378,51 @@ class PackagesController < ApplicationController packages end + # Text search for packages: an exact package name match wins without + # touching the vector service. Otherwise packages whose name starts + # with the query (or contains all query tokens, catching compound + # words like "sqlite browser" -> "sqlitebrowser") are promoted above + # the semantic (vector) nearest neighbor results. Classic PostgreSQL + # full-text search kicks in when the vector service cannot be reached. + MAX_PROMOTED_NAME_MATCHES = 8 + MAX_SEMANTIC_RESULTS = 100 + private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS + + def search_packages(query) + exact_match = Package.find_by(name: query) + return Package.where(id: exact_match.id) if exact_match + + tokens = query.to_s.split(/\s+/) + name_matches = ( + Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) + + Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES) + ).uniq(&:id) + + results = (name_matches + semantic_results(query)).uniq(&:id) + + # Queries that share no vocabulary with our package data can produce + # no meaningful results. Skip the vector service round-trip and show + # an honest empty result instead of random packages. + results.empty? ? Package.none : results + end + + # Semantic (vector) nearest neighbor search with a full-text fallback + # for when the external embedding service is unreachable. + def semantic_results(query) + unless Package.lexical_overlap?(query) + logger.debug "Query '#{query}' has no lexical overlap with package data" + return [] + end + + begin + Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a + rescue Vectorizer::Error => e + logger.error "Semantic search failed: #{e.message}" + flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' + Package.general_search(query).limit(100).to_a + end + end + # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/models/package.rb b/app/models/package.rb index 441f5c4..e2e1b00 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -2,6 +2,9 @@ require 'open-uri' # allows to load URLs using open() require 'json' require 'deb_importer' +# Search and embedding logic keeps accumulating here; the class is +# cohesive enough that splitting it is not worth the indirection. +# rubocop:disable Metrics/ClassLength class Package < ApplicationRecord # PostgreSQL-based full-text search: # https://github.com/Casecommons/pg_search @@ -26,6 +29,11 @@ class Package < ApplicationRecord inverse_of: :package, dependent: :destroy + # Nearest neighbor search using pgvector. The "embedding" column stores + # a vector computed from description + long_description by an external + # web service (see Vectorizer and lib/tasks/vectorize_packages.rake). + has_neighbors :embedding + # default_scope { # order('name ASC') # } @@ -40,6 +48,94 @@ class Package < ApplicationRecord long_description.split(/\n\.\n/).first if long_description end + # The text that the embedding vector is computed from: description + # and long description combined. + def embedding_text + [description, long_description].compact.join("\n") + end + + # Compute and store the embedding vector for this package by asking + # the external vector service. Raises Vectorizer::Error on failure. + # Returns true if the vector was saved. + def update_embedding! + text = embedding_text + return false if text.blank? + + self.embedding = Vectorizer.embed(text) + save! + end + + # Return a relation of packages whose stored embeddings are closest to + # the given query vector (an array of floats). Records get a + # "neighbor_distance" attribute added (cosine distance, lower is better). + # The returned relation can still be chained (e.g. filtered or paginated). + # + # Only returns packages that already have an embedding computed. + def self.nearest_to_vector(vector, limit: nil) + neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine') + limit ? neighbors.limit(limit) : neighbors + end + + # Turn a free-text search string into a vector via the external web + # service and find the closest packages. + def self.nearest_to_text(text, limit: nil) + return none if text.blank? + + nearest_to_vector(Vectorizer.embed(text), limit: limit) + end + + # Packages whose name starts with the given string - used to promote + # obvious matches like "sqlite3" for a "sqlite" search. + def self.name_starts_with(text) + return none if text.blank? + + where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name) + end + + # Packages whose name contains all given tokens as substrings. Catches + # compound words split by the user, e.g. "sqlite browser" finding the + # "sqlitebrowser" package. + def self.name_contains_all(tokens) + tokens = tokens.select { |token| token.length >= 2 } + return none if tokens.empty? + + scope = self + tokens.each do |token| + scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\') + end + scope.order(:name) + end + + def self.escape_for_like(text) + # Escape LIKE wildcards so that package name characters like "+" or "." + # cannot break the pattern + text.gsub(/[\\%_]/) { |char| "\\#{char}" } + end + + # Check whether the given text shares at least one lexeme with any + # package name/description in the database. Queries without such an + # overlap (gibberish, keyboard mash, unsupported languages) can + # neither match full-text nor produce meaningful semantic results - + # the embedding model maps unknown tokens close to the corpus + # centroid, making them look like good matches. + def self.lexical_overlap?(text) + tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) } + return false if tokens.empty? + + # Pin the same text search dictionary that the packages_fts index + # uses - the server-wide default can be anything. + tsquery = tokens.map do |token| + sanitize_sql(["plainto_tsquery('english', ?)", token]) + end.join(' || ') + # The expression must match the packages_fts GIN index definition + # so that Postgres can use the index. + where( + "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \ + "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \ + "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})" + ).exists? + end + # Return a query of all packages that have screenshots def self.with_screenshots # Query for all packages who's ID appears in a screenshot's "package_id" field diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 84d8599..53009da 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -14,7 +14,7 @@ - if @packages.any? - if @view_style==:grid .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium" - - @packages.all.each do |pkg| + - @packages.each do |pkg| .cell.pkgcard data-equalizer-watch=true a.black href=package_path(name: pkg.name) .image diff --git a/config/initializers/will_paginate.rb b/config/initializers/will_paginate.rb new file mode 100644 index 0000000..e36b7b3 --- /dev/null +++ b/config/initializers/will_paginate.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Enable will_paginate's Array#paginate so that merged search results +# (name prefix matches + semantic results) can be paginated like +# ActiveRecord relations. +require 'will_paginate/array' diff --git a/db/migrate/20260821203833_install_neighbor_vector.rb b/db/migrate/20260821203833_install_neighbor_vector.rb new file mode 100644 index 0000000..d591f1c --- /dev/null +++ b/db/migrate/20260821203833_install_neighbor_vector.rb @@ -0,0 +1,5 @@ +class InstallNeighborVector < ActiveRecord::Migration[8.1] + def change + enable_extension "vector" + end +end diff --git a/db/migrate/20260821210000_add_embedding_to_packages.rb b/db/migrate/20260821210000_add_embedding_to_packages.rb new file mode 100644 index 0000000..62825f9 --- /dev/null +++ b/db/migrate/20260821210000_add_embedding_to_packages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# Adds a vector column to store text embeddings of the package +# description and long description. The vectors are computed by an +# external web service and used for nearest neighbor search. +# +# The pgvector extension is enabled by the InstallNeighborVector migration. +class AddEmbeddingToPackages < ActiveRecord::Migration[8.1] + def change + add_column :packages, :embedding, :vector, limit: 384 + + # HNSW index for fast approximate nearest neighbor searches + add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops + end +end diff --git a/db/schema.rb b/db/schema.rb index 2c57b69..a2ad488 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,133 +10,131 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do - # These are extensions that must be enabled to support this database - enable_extension 'pg_catalog.plpgsql' +ActiveRecord::Schema[8.1].define(version: 2026_08_21_210000) do + # These are extensions that must be enabled in order to support this database + enable_extension "pg_catalog.plpgsql" + enable_extension "vector" - create_table 'action_mailbox_inbound_emails', force: :cascade do |t| - t.integer 'status', default: 0, null: false - t.string 'message_id', null: false - t.string 'message_checksum', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['message_id', 'message_checksum'], -name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true + create_table "action_mailbox_inbound_emails", force: :cascade do |t| + t.datetime "created_at", null: false + t.string "message_checksum", null: false + t.string "message_id", null: false + t.integer "status", default: 0, null: false + t.datetime "updated_at", null: false + t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true end - create_table 'action_text_rich_texts', force: :cascade do |t| - t.string 'name', null: false - t.text 'body' - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.datetime 'created_at', null: false - t.datetime 'updated_at', null: false - t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', -unique: true + create_table "action_text_rich_texts", force: :cascade do |t| + t.text "body" + t.datetime "created_at", null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.datetime "updated_at", null: false + t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true end - create_table 'active_storage_attachments', force: :cascade do |t| - t.string 'name', null: false - t.string 'record_type', null: false - t.bigint 'record_id', null: false - t.bigint 'blob_id', null: false - t.datetime 'created_at', precision: nil, null: false - t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id' - t.index ['record_type', 'record_id', 'name', 'blob_id'], -name: 'index_active_storage_attachments_uniqueness', unique: true + create_table "active_storage_attachments", force: :cascade do |t| + t.bigint "blob_id", null: false + t.datetime "created_at", precision: nil, null: false + t.string "name", null: false + t.bigint "record_id", null: false + t.string "record_type", null: false + t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" + t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true end - create_table 'active_storage_blobs', force: :cascade do |t| - t.string 'key', null: false - t.string 'filename', null: false - t.string 'content_type' - t.text 'metadata' - t.bigint 'byte_size', null: false - t.string 'checksum' - t.datetime 'created_at', precision: nil, null: false - t.string 'service_name', null: false - t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true + create_table "active_storage_blobs", force: :cascade do |t| + t.bigint "byte_size", null: false + t.string "checksum" + t.string "content_type" + t.datetime "created_at", precision: nil, null: false + t.string "filename", null: false + t.string "key", null: false + t.text "metadata" + t.string "service_name", null: false + t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true end - create_table 'active_storage_variant_records', force: :cascade do |t| - t.bigint 'blob_id', null: false - t.string 'variation_digest', null: false - t.index ['blob_id', 'variation_digest'], -name: 'index_active_storage_variant_records_uniqueness', unique: true + create_table "active_storage_variant_records", force: :cascade do |t| + t.bigint "blob_id", null: false + t.string "variation_digest", null: false + t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true end - create_table 'logs', id: :serial, force: :cascade do |t| - t.string 'message' - t.string 'level' - t.string 'section' - t.inet 'ip_address' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.integer 'user_id' - t.integer 'package_id' - t.integer 'screenshot_id' + create_table "logs", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil, null: false + t.inet "ip_address" + t.string "level" + t.string "message" + t.integer "package_id" + t.integer "screenshot_id" + t.string "section" + t.datetime "updated_at", precision: nil, null: false + t.integer "user_id" end - create_table 'packages', id: :serial, force: :cascade do |t| - t.string 'name', null: false - t.string 'description', limit: 80 - t.string 'section', limit: 50 - t.string 'maintainer', limit: 100 - t.string 'maintainer_email', limit: 100 - t.string 'homepage', limit: 400 - t.string 'version', limit: 200 - t.text 'long_description' - t.string 'origin', limit: 80 - t.datetime 'created_at', precision: nil - t.datetime 'updated_at', precision: nil - t.integer 'visits', default: 0 - t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", -name: 'packages_fts', using: :gin - t.unique_constraint ['name'], name: 'packages_name_key' + create_table "packages", id: :serial, force: :cascade do |t| + t.datetime "created_at", precision: nil + t.string "description", limit: 80 + t.vector "embedding", limit: 384 + t.string "homepage", limit: 400 + t.text "long_description" + t.string "maintainer", limit: 100 + t.string "maintainer_email", limit: 100 + t.string "name", null: false + t.string "origin", limit: 80 + t.string "section", limit: 50 + t.datetime "updated_at", precision: nil + t.string "version", limit: 200 + t.integer "visits", default: 0 + t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin + t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw + t.unique_constraint ["name"], name: "packages_name_key" end - create_table 'screenshots', id: :serial, force: :cascade do |t| - t.integer 'package_id' - t.string 'version', limit: 50 - t.datetime 'created_at', precision: nil - t.string 'uploaderhash', limit: 72 - t.boolean 'approved', default: false, null: false - t.text 'description' - t.datetime 'updated_at', precision: nil - t.string 'image_fingerprint' - t.integer 'user_id', default: 0 - t.text 'simage_data' - t.inet 'uploaderip' - t.boolean 'hidden', default: false - t.index ['id', 'approved'], name: 'id_approved' - t.index ['id', 'uploaderhash'], name: 'id_uploaderhash' + create_table "screenshots", id: :serial, force: :cascade do |t| + t.boolean "approved", default: false, null: false + t.datetime "created_at", precision: nil + t.text "description" + t.boolean "hidden", default: false + t.string "image_fingerprint" + t.integer "package_id" + t.text "simage_data" + t.datetime "updated_at", precision: nil + t.string "uploaderhash", limit: 72 + t.inet "uploaderip" + t.integer "user_id", default: 0 + t.string "version", limit: 50 + t.index ["id", "approved"], name: "id_approved" + t.index ["id", "uploaderhash"], name: "id_uploaderhash" end - create_table 'users', id: :serial, force: :cascade do |t| - t.text 'name' - t.datetime 'created_at', precision: nil, null: false - t.datetime 'updated_at', precision: nil, null: false - t.string 'email', default: '', null: false - t.string 'encrypted_password', default: '', null: false - t.integer 'sign_in_count', default: 0, null: false - t.datetime 'current_sign_in_at', precision: nil - t.datetime 'last_sign_in_at', precision: nil - t.inet 'current_sign_in_ip' - t.inet 'last_sign_in_ip' - t.integer 'failed_attempts', default: 0, null: false - t.string 'unlock_token' - t.datetime 'locked_at', precision: nil - t.string 'provider' - t.string 'uid' - t.boolean 'admin_role', default: false - t.boolean 'moderator_role', default: false - t.boolean 'pseudo', default: false - t.integer 'approved_screenshots', default: 0 - t.integer 'rejected_screenshots', default: 0 - t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true + create_table "users", id: :serial, force: :cascade do |t| + t.boolean "admin_role", default: false + t.integer "approved_screenshots", default: 0 + t.datetime "created_at", precision: nil, null: false + t.datetime "current_sign_in_at", precision: nil + t.inet "current_sign_in_ip" + t.string "email", default: "", null: false + t.string "encrypted_password", default: "", null: false + t.integer "failed_attempts", default: 0, null: false + t.datetime "last_sign_in_at", precision: nil + t.inet "last_sign_in_ip" + t.datetime "locked_at", precision: nil + t.boolean "moderator_role", default: false + t.text "name" + t.string "provider" + t.boolean "pseudo", default: false + t.integer "rejected_screenshots", default: 0 + t.integer "sign_in_count", default: 0, null: false + t.string "uid" + t.string "unlock_token" + t.datetime "updated_at", precision: nil, null: false + t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true end - add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id' - add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey' + add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" + add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" + add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey" end diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake new file mode 100644 index 0000000..1c19837 --- /dev/null +++ b/lib/tasks/vectorize_packages.rake @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +namespace :debshots do + desc 'Compute embedding vectors for all packages based on their description ' \ + 'and long description (missing ones only unless FORCE=1). ' \ + 'Set CONCURRENCY=n to control parallel HTTP requests.' + task compute_vectors: :environment do + logger = Logger.new($stdout) + logger.level = Logger::INFO + + force = ENV['FORCE'].present? + concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max + + # Never use more threads than the connection pool has connections. + # Keep one connection reserved for the main thread. + pool_size = ActiveRecord::Base.connection_pool.size + concurrency = [concurrency, pool_size - 1].min + concurrency = 0 if concurrency.negative? + + scope = force ? Package.all : Package.where(embedding: nil) + total = scope.count + + if total.zero? + logger.info 'No packages need vector computation.' + next + end + + logger.info "Computing vectors for #{total} packages " \ + "(#{force ? 'recomputing' : 'missing only'}, " \ + "service: #{Vectorizer.service_url}, " \ + "concurrency: #{concurrency.zero? ? 1 : concurrency})" + + stats_mutex = Mutex.new + stats = { done: 0, failed: 0 } + + process_package = lambda do |package| + stats_mutex.synchronize do + begin + raise ArgumentError, 'no description text' if package.embedding_text.blank? + + package.update_embedding! + stats[:done] += 1 + rescue Vectorizer::Error, ArgumentError => e + stats[:failed] += 1 + logger.error "Failed for package #{package.name}: #{e.message}" + end + + processed = stats[:done] + stats[:failed] + return unless (processed % 100).zero? + + percent = (processed.to_f / total * 100).round(1) + logger.info "Progress: #{processed}/#{total} (#{percent}%)" + end + end + + if concurrency.zero? + # Sequential fallback when the connection pool has no room for workers + scope.find_each { |package| process_package.call(package) } + else + # Feed package IDs into a queue that the worker threads pull from. + work_queue = Queue.new + scope.pluck(:id).each { |id| work_queue << id } + concurrency.times { work_queue << nil } # one poison pill per worker + + workers = Array.new(concurrency) do + Thread.new do + # Each thread gets its own database connection from the pool. + ActiveRecord::Base.connection_pool.with_connection do + while (package_id = work_queue.pop) + package = Package.find_by(id: package_id) + process_package.call(package) if package + end + end + rescue StandardError => e + logger.error "Worker crashed: #{e.message}" + end + end + + workers.each(&:join) + end + + logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures." + end +end diff --git a/lib/vectorizer.rb b/lib/vectorizer.rb new file mode 100644 index 0000000..84df3c4 --- /dev/null +++ b/lib/vectorizer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'net/http' +require 'json' + +# Client for the external web service that turns text into embedding +# vectors for nearest neighbor search (pgvector). +# +# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...] +# +# The service URL can be overridden with the VECTOR_SERVICE_URL +# environment variable. +class Vectorizer + # Raised when the vector service cannot be reached or returns garbage + class Error < StandardError; end + + DEFAULT_URL = 'https://vector-search.workaround.org/vector' + + class << self + # Compute the embedding vector for the given text. + # Returns an array of floats. + def embed(text) + new.embed(text) + end + + def service_url + ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL) + end + end + + def embed(text) + response = post_text(text) + + unless response.is_a?(Net::HTTPSuccess) + raise Error, "Vector service returned HTTP #{response.code}" + end + + JSON.parse(response.body).fetch('vector') + rescue JSON::ParserError => e + raise Error, "Vector service returned invalid JSON: #{e.message}" + rescue KeyError + raise Error, 'Vector service response did not contain a "vector" field' + end + + private + + def post_text(text) + uri = URI.parse(self.class.service_url) + request = build_request(uri, text) + + options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 } + Net::HTTP.start(uri.host, uri.port, **options) do |http| + http.request(request) + end + rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e + raise Error, "Could not reach vector service: #{e.message}" + end + + def build_request(uri, text) + request = Net::HTTP::Post.new(uri.request_uri) + request['Content-Type'] = 'application/json' + request.body = { text: text }.to_json + + # Optional Bearer token if the service runs with API_SECRET set + secret = ENV['VECTOR_SERVICE_SECRET'] + request['Authorization'] = "Bearer #{secret}" if secret.present? + + request + end +end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index c9eae31..4d89d88 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'test_helper' class PackagesControllerTest < ActionController::TestCase @@ -20,6 +22,81 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'A program to browse web sites.' end + test 'exact package name search wins without calling the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'vim' } + assert_response :success + assert_select 'div', 'vim' + end + end + + test 'gibberish search shows no results and never calls the vector service' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text) { flunk 'should not call vector search' }) do + get :grid, params: { search: 'asdfgh zzzz' } + assert_response :success + # the grid renders every package in the database when no search + # filter applies - so an empty result must not fall back to that + assert_select 'div', text: /package-\d+/, count: 0 + end + end + + test 'name prefix matches rank above semantic results' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do + get :grid, params: { search: 'package' } + assert_response :success + # fixtures create package-0..package-99 - prefix hits must render + assert_select 'div', 'package-0' + # ...and before any semantically found package + assert response.body.index('package-0') < response.body.index('vim') + end + end + + test 'prefix matches and semantic results are deduplicated' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do + get :grid, params: { search: 'fire' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + end + end + + test 'compound words in search match packages with joined names' do + # "fire fox" should find "firefox" via name substring matching + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.none }) do + get :grid, params: { search: 'fire fox' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + end + end + + test 'search uses semantic (vector) results' do + relation = Package.where(name: packages(:vim).name) + with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + end + end + + # There is deliberately no fallback for an empty vector search + # result: KNN always returns something as long as packages have + # embeddings. Full-text search is only used when the vector service + # cannot be reached. + + test 'search falls back to full-text search when vector service fails' do + with_stubbed_method(Vectorizer, :embed, + ->(_text) { raise Vectorizer::Error, 'service down' }) do + get :grid, params: { search: 'editor' } + assert_response :success + assert_select 'div', 'vim' + assert flash[:error].present? + end + end + # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') diff --git a/test/models/package_test.rb b/test/models/package_test.rb index 124f6c4..36562a2 100644 --- a/test/models/package_test.rb +++ b/test/models/package_test.rb @@ -1,7 +1,107 @@ +# frozen_string_literal: true + require 'test_helper' class PackageTest < ActiveSupport::TestCase - # test "the truth" do - # assert true - # end + test 'embedding_text combines description and long description' do + package = packages(:firefox) + assert_equal "A web browser\nA program to browse web sites.", package.embedding_text + end + + test 'embedding_text works with missing long description' do + package = packages(:firefox) + package.long_description = nil + assert_equal 'A web browser', package.embedding_text + end + + test 'update_embedding! stores the vector returned by the vector service' do + vector = Array.new(384) { |i| i / 384.0 } + with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do + package = packages(:vim) + assert package.update_embedding! + # The vector type stores float32 so allow small rounding differences + package.reload.embedding.to_a.each_with_index do |actual, i| + assert_in_delta vector[i], actual, 1e-6 + end + end + end + + test 'update_embedding! does nothing without description text' do + package = packages(:firefox) + package.description = nil + package.long_description = nil + with_stubbed_method(Vectorizer, :embed, + ->(_text) { flunk 'should not call the vector service' }) do + assert_not package.update_embedding! + end + end + + test 'nearest_to_text returns none for blank text' do + assert_empty Package.nearest_to_text('') + assert_empty Package.nearest_to_text(nil) + end + + test 'name_starts_with matches name prefixes case-insensitively' do + assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name + assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name + assert_empty Package.name_starts_with('') + assert_empty Package.name_starts_with(nil) + end + + test 'name_contains_all finds compound words split by spaces' do + assert_includes Package.name_contains_all(%w[fire fox]).map(&:name), + packages(:firefox).name + # all tokens must be present in the name + assert_empty Package.name_contains_all(%w[firefox zzzz]) + assert_empty Package.name_contains_all([]) + # single character tokens would flood results - ignore them + assert_empty Package.name_contains_all(['f']) + end + + test 'lexical_overlap? pins the english dictionary regardless of server config' do + # Regression: with default_text_search_config = german (as on some + # servers) 'browser' stems to 'brows' which never matches the + # english-stemmed index. + connection = ActiveRecord::Base.connection + original = connection.select_value('SHOW default_text_search_config') + connection.execute("SET default_text_search_config = 'german'") + assert Package.lexical_overlap?('browser') + ensure + connection&.execute("SET default_text_search_config = '#{original}'") + end + + test 'lexical_overlap? detects real words vs gibberish' do + # words present in fixture package descriptions + assert Package.lexical_overlap?('browser') + assert Package.lexical_overlap?('email client for vim syntax') + assert Package.lexical_overlap?('the browser') # stopwords are ignored + # gibberish, numbers and empty strings share no vocabulary + assert_not Package.lexical_overlap?('asdfgh zzzz') + assert_not Package.lexical_overlap?('123 !!!') + assert_not Package.lexical_overlap?('the') # stopword-only query + assert_not Package.lexical_overlap?('') + assert_not Package.lexical_overlap?(nil) + end + + test 'nearest_to_text embeds the query and searches' do + vector = Array.new(384, 0.25) + embed_called_with = nil + with_stubbed_method(Vectorizer, :embed, + lambda { |text| + embed_called_with = text + vector + }) do + received = [] + results = with_stubbed_method(Package, :nearest_neighbors, + lambda { |_field, received_vector, **_opts| + received << received_vector + [packages(:vim)] + }) do + Package.nearest_to_text('some search text') + end + assert_equal [packages(:vim)], results + assert_equal 'some search text', embed_called_with + assert_equal [vector], received + end + end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 4889a19..1d786a4 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) require 'rails/test_help' @@ -15,6 +17,23 @@ module ActiveSupport class TestCase # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. fixtures :all + + # Replace a class-level method with a fixed return value or lambda + # for the duration of the block. Used to keep tests away from + # external services like the vector embedding web service. + def with_stubbed_method(klass, method_name, value_or_lambda) + original = klass.method(method_name) + klass.define_singleton_method(method_name) do |*args, **kwargs| + if value_or_lambda.respond_to?(:call) + value_or_lambda.call(*args, **kwargs) + else + value_or_lambda + end + end + yield + ensure + klass.define_singleton_method(method_name, original) + end end end From fdf3289f5438f3a2c537ae4a43ec2f75eb25e697 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sat, 22 Aug 2026 22:08:05 +0200 Subject: [PATCH 02/22] Remove lograge configuration from Docker image and production --- Dockerfile | 14 ++++++-------- config/environments/production.rb | 9 --------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 177771d..91a0c7c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ARG BUN_VERSION=1.3.9 # --- Stage 1: Grab Node.js Binaries --- # We use the official image as a source for pre-compiled binaries -FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source +FROM docker.io/library/node:${NODE_VERSION} AS node_source # --- Stage 2: Base (Runtime Environment) --- FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base @@ -36,15 +36,13 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ apt-get install --no-install-recommends -y \ - build-essential git libyaml-dev node-gyp pkg-config python-is-python3 + build-essential git libyaml-dev pkg-config python-is-python3 -# "Heist" Node.js and npm from the node_source stage (build-time only) -COPY --from=node_source /usr/local/bin/node /usr/local/bin/node -COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules +# Install Node.js and npm from node_source +COPY --from=node_source /usr/local/ /usr/local/ -# Re-link npm and install Bun (build-time only) -RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ - npm install -g bun@${BUN_VERSION} +# Install Bun +RUN npm install -g bun@${BUN_VERSION} # 1. Install Gems (Separate COPY for maximum caching) COPY Gemfile Gemfile.lock ./ diff --git a/config/environments/production.rb b/config/environments/production.rb index ced9b3c..b1c0f32 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -37,15 +37,6 @@ Rails.application.configure do config.log_tags = [:request_id] config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) - # Lograge - structured request logging - config.lograge.enabled = true - config.lograge.custom_options = lambda do |event| - { - request_id: event.payload[:request_id], - user_id: event.payload[:user_id] - } - end - # Change to "debug" to log everything (including potentially personally-identifiable information!). config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info') From 4d07f43a2e8f4a535c429f707bf50b4575f3f8bc Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sat, 22 Aug 2026 23:47:42 +0200 Subject: [PATCH 03/22] Show related packages based on description embeddings Package pages get a "Related packages" section: a nearest neighbor query against the package's own stored embedding (no external service call), excluding the package itself. Packages without an embedding render no section. --- README.md | 4 ++++ app/controllers/packages_controller.rb | 3 +++ app/models/package.rb | 11 +++++++++++ app/views/packages/details.slim | 13 +++++++++++++ test/controllers/packages_controller_test.rb | 15 +++++++++++++++ 5 files changed, 46 insertions(+) diff --git a/README.md b/README.md index deaef3a..365d995 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ Queries that share no vocabulary with the package data (gibberish, keyboard mash) return an honest empty result instead of random semantic matches. +Package pages also show semantically **related packages**, computed as +a nearest neighbor query against the package's own description +embedding - no external service call involved. + ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 726f4f0..5c65c97 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -30,6 +30,9 @@ class PackagesController < ApplicationController @screenshots = @package.screenshots.accessible_by(current_ability, :view).paginate( page: @page, per_page: 6 ) + # Semantic recommendations based on the package's own description + # embedding - a plain nearest neighbor query, no service call. + @related_packages = @package.related_packages end end diff --git a/app/models/package.rb b/app/models/package.rb index e2e1b00..71b89c7 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -112,6 +112,16 @@ class Package < ApplicationRecord text.gsub(/[\\%_]/) { |char| "\\#{char}" } end + # Packages semantically similar to this one, based on the stored + # description embedding. Returns an empty relation for packages that + # have no embedding (yet). + def related_packages(limit: 6) + return Package.none unless embedding + + Package.nearest_to_vector(embedding, limit: limit + 1) + .where.not(id: id).limit(limit) + end + # Check whether the given text shares at least one lexeme with any # package name/description in the database. Queries without such an # overlap (gibberish, keyboard mash, unsupported languages) can @@ -209,3 +219,4 @@ class Package < ApplicationRecord end end end +# rubocop:enable Metrics/ClassLength diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index 03c9609..7443302 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -70,6 +70,19 @@ / #metadata is the anchor for this sticky container. = render(partial: 'details_rightbox', locals: {pkg: @package}) + / Semantically similar packages, based on the package's description embedding + - if @related_packages.any? + #related + h3 Related packages + .grid-x.grid-margin-x.medium-up-2.large-up-3 data-equalizer=true + - @related_packages.each do |pkg| + .cell.pkgcard data-equalizer-watch=true + a.black href=package_path(name: pkg.name) + .text.pkgname + = pkg.name + .text.small + = pkg.description + javascript: // Where to send POST requests for uploads let ajax_upload_url = '#{{upload_receive_json_path}}'; diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index 4d89d88..92a6ea1 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -73,6 +73,21 @@ class PackagesControllerTest < ActionController::TestCase end end + test 'details shows semantically related packages' do + packages(:vim).update_column(:embedding, Array.new(384, 0.25)) + packages(:firefox).update_column(:embedding, Array.new(384, 0.75)) + + get :details, params: { name: 'vim' } + assert_response :success + assert_select '#related .pkgname', text: 'firefox' + end + + test 'details without embedding renders no related section' do + get :details, params: { name: 'vim' } + assert_response :success + assert_select '#related', count: 0 + end + test 'search uses semantic (vector) results' do relation = Package.where(name: packages(:vim).name) with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do From 3550a4e2e2b53582b02d7ff875c236594698e449 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 00:11:52 +0200 Subject: [PATCH 04/22] Harden vectorization task against per-package failures A single failing package used to kill its whole worker thread silently because only Vectorizer::Error was rescued per package. Now every StandardError is caught and logged with class and backtrace, and the embedding write is narrowed to update_column(:embedding) so the task only ever issues a minimal UPDATE - it cannot touch any other column or fire model callbacks. Progress stats no longer serialize database writes through the mutex. --- app/models/package.rb | 5 ++--- lib/tasks/vectorize_packages.rake | 26 ++++++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/app/models/package.rb b/app/models/package.rb index 71b89c7..c2e31de 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -56,13 +56,12 @@ class Package < ApplicationRecord # Compute and store the embedding vector for this package by asking # the external vector service. Raises Vectorizer::Error on failure. - # Returns true if the vector was saved. + # Returns true if a vector was saved. def update_embedding! text = embedding_text return false if text.blank? - self.embedding = Vectorizer.embed(text) - save! + update_column(:embedding, Vectorizer.embed(text)) end # Return a relation of packages whose stored embeddings are closest to diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake index 1c19837..ce873d8 100644 --- a/lib/tasks/vectorize_packages.rake +++ b/lib/tasks/vectorize_packages.rake @@ -1,5 +1,8 @@ # frozen_string_literal: true +# Long rake task bodies are normal; the block-length cop is meant for +# application code. +# rubocop:disable Metrics/BlockLength namespace :debshots do desc 'Compute embedding vectors for all packages based on their description ' \ 'and long description (missing ones only unless FORCE=1). ' \ @@ -34,17 +37,20 @@ namespace :debshots do stats = { done: 0, failed: 0 } process_package = lambda do |package| + begin + raise ArgumentError, 'no description text' if package.embedding_text.blank? + + package.update_embedding! + stats_mutex.synchronize { stats[:done] += 1 } + rescue StandardError => e + # A single broken package must never take down a whole worker + # thread - log enough detail to diagnose it and move on. + stats_mutex.synchronize { stats[:failed] += 1 } + logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}" + logger.error e.backtrace.first(3).join("\n") if e.backtrace + end + stats_mutex.synchronize do - begin - raise ArgumentError, 'no description text' if package.embedding_text.blank? - - package.update_embedding! - stats[:done] += 1 - rescue Vectorizer::Error, ArgumentError => e - stats[:failed] += 1 - logger.error "Failed for package #{package.name}: #{e.message}" - end - processed = stats[:done] + stats[:failed] return unless (processed % 100).zero? From 2b79470feb25f22e04fad50345fa6888ee053d19 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 00:13:50 +0200 Subject: [PATCH 05/22] Simplify vectorization task to sequential processing Drop the threaded worker pool (queue, mutexes, connection pooling). The task now processes packages one by one via find_each - plenty fast since the external embedding service dominates latency, and much easier to reason about. --- lib/tasks/vectorize_packages.rake | 67 +++++++------------------------ 1 file changed, 15 insertions(+), 52 deletions(-) diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake index ce873d8..4fbfe7b 100644 --- a/lib/tasks/vectorize_packages.rake +++ b/lib/tasks/vectorize_packages.rake @@ -5,21 +5,12 @@ # rubocop:disable Metrics/BlockLength namespace :debshots do desc 'Compute embedding vectors for all packages based on their description ' \ - 'and long description (missing ones only unless FORCE=1). ' \ - 'Set CONCURRENCY=n to control parallel HTTP requests.' + 'and long description (missing ones only unless FORCE=1)' task compute_vectors: :environment do logger = Logger.new($stdout) logger.level = Logger::INFO force = ENV['FORCE'].present? - concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max - - # Never use more threads than the connection pool has connections. - # Keep one connection reserved for the main thread. - pool_size = ActiveRecord::Base.connection_pool.size - concurrency = [concurrency, pool_size - 1].min - concurrency = 0 if concurrency.negative? - scope = force ? Package.all : Package.where(embedding: nil) total = scope.count @@ -30,61 +21,33 @@ namespace :debshots do logger.info "Computing vectors for #{total} packages " \ "(#{force ? 'recomputing' : 'missing only'}, " \ - "service: #{Vectorizer.service_url}, " \ - "concurrency: #{concurrency.zero? ? 1 : concurrency})" + "service: #{Vectorizer.service_url})" - stats_mutex = Mutex.new - stats = { done: 0, failed: 0 } + done = 0 + failed = 0 - process_package = lambda do |package| + scope.find_each do |package| begin raise ArgumentError, 'no description text' if package.embedding_text.blank? package.update_embedding! - stats_mutex.synchronize { stats[:done] += 1 } + done += 1 rescue StandardError => e - # A single broken package must never take down a whole worker - # thread - log enough detail to diagnose it and move on. - stats_mutex.synchronize { stats[:failed] += 1 } + # A single broken package must never stop the whole run - log + # enough detail to diagnose it and move on. + failed += 1 logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}" logger.error e.backtrace.first(3).join("\n") if e.backtrace end - stats_mutex.synchronize do - processed = stats[:done] + stats[:failed] - return unless (processed % 100).zero? + processed = done + failed + next unless (processed % 100).zero? || processed == total - percent = (processed.to_f / total * 100).round(1) - logger.info "Progress: #{processed}/#{total} (#{percent}%)" - end + percent = (processed.to_f / total * 100).round(1) + logger.info "Progress: #{processed}/#{total} (#{percent}%)" end - if concurrency.zero? - # Sequential fallback when the connection pool has no room for workers - scope.find_each { |package| process_package.call(package) } - else - # Feed package IDs into a queue that the worker threads pull from. - work_queue = Queue.new - scope.pluck(:id).each { |id| work_queue << id } - concurrency.times { work_queue << nil } # one poison pill per worker - - workers = Array.new(concurrency) do - Thread.new do - # Each thread gets its own database connection from the pool. - ActiveRecord::Base.connection_pool.with_connection do - while (package_id = work_queue.pop) - package = Package.find_by(id: package_id) - process_package.call(package) if package - end - end - rescue StandardError => e - logger.error "Worker crashed: #{e.message}" - end - end - - workers.each(&:join) - end - - logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures." + logger.info "Done. #{done} vectors computed, #{failed} failures." end end +# rubocop:enable Metrics/BlockLength From bada261841050199f8f8da0f54a273b2667f795d Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 00:20:30 +0200 Subject: [PATCH 06/22] Skip packages without description text in vectorization Packages with neither description nor long description cannot be embedded at all; they used to fail on every run. Exclude them from the scope instead. --- lib/tasks/vectorize_packages.rake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake index 4fbfe7b..66bf7a2 100644 --- a/lib/tasks/vectorize_packages.rake +++ b/lib/tasks/vectorize_packages.rake @@ -11,7 +11,11 @@ namespace :debshots do logger.level = Logger::INFO force = ENV['FORCE'].present? - scope = force ? Package.all : Package.where(embedding: nil) + # Packages without any description text cannot be embedded at all - + # exclude them so they do not fail on every single run. + base_scope = Package.where("COALESCE(description, '') <> '' OR " \ + "COALESCE(long_description, '') <> ''") + scope = force ? base_scope : base_scope.where(embedding: nil) total = scope.count if total.zero? From 61b64fae2550d78ece946258e285ad4d24d57bff Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 00:31:04 +0200 Subject: [PATCH 07/22] Log the failing SQL statement in vectorization task Unique violations on packages_name_key happen on a statement that cannot logically touch name - log the exact query and a longer backtrace to identify what really fails. --- lib/tasks/vectorize_packages.rake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake index 66bf7a2..aee5d45 100644 --- a/lib/tasks/vectorize_packages.rake +++ b/lib/tasks/vectorize_packages.rake @@ -30,6 +30,13 @@ namespace :debshots do done = 0 failed = 0 + # Remember the most recent SQL statement so failures can be traced + # to the exact query that triggered them. + last_sql = nil + ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| + last_sql = args.last[:sql] + end + scope.find_each do |package| begin raise ArgumentError, 'no description text' if package.embedding_text.blank? @@ -41,7 +48,8 @@ namespace :debshots do # enough detail to diagnose it and move on. failed += 1 logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}" - logger.error e.backtrace.first(3).join("\n") if e.backtrace + logger.error "Last SQL: #{last_sql}" + logger.error e.backtrace.first(15).join("\n") if e.backtrace end processed = done + failed From 323b7004edfe3bdec334e51fed2f521c2efe3b95 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 10:39:50 +0200 Subject: [PATCH 08/22] Expose which search strategy produced each result query_packages now assigns @search_sources - a hash mapping package ids to the matching strategy that put them into the results: :exact, :name_prefix, :name_contains, :semantic or :fulltext. The grid and list views render a small badge with a humanized label per result, making the search tiers observable. semantic_results() was folded into search_packages() so each branch tags its own source. Adds the rails-controller-testing gem for assigns() in tests. --- Gemfile | 2 + Gemfile.lock | 5 ++ app/controllers/packages_controller.rb | 55 +++++++++++--------- app/helpers/packages_helper.rb | 12 +++++ app/views/packages/browse.slim | 6 +++ test/controllers/packages_controller_test.rb | 13 +++++ 6 files changed, 69 insertions(+), 24 deletions(-) diff --git a/Gemfile b/Gemfile index 6e1b32d..c7a6922 100644 --- a/Gemfile +++ b/Gemfile @@ -108,6 +108,8 @@ group :test do # Use system testing [https://guides.rubyonrails.org/testing.html#system-testing] gem 'capybara' gem 'selenium-webdriver' + # Provides the assigns() test helper extracted from modern Rails + gem 'rails-controller-testing' end # TODO… https://github.com/galetahub/simple-captcha diff --git a/Gemfile.lock b/Gemfile.lock index 9cf75ac..65bbd8a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -381,6 +381,10 @@ GEM activesupport (= 8.1.3) bundler (>= 1.15.0) railties (= 8.1.3) + rails-controller-testing (1.0.5) + actionpack (>= 5.0.1.rc1) + actionview (>= 5.0.1.rc1) + activesupport (>= 5.0.1.rc1) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest @@ -592,6 +596,7 @@ DEPENDENCIES propshaft puma (>= 5.0) rails (~> 8.1.0) + rails-controller-testing rails-erd rails-healthcheck rails_icons (~> 1.3) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 5c65c97..eec639d 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -362,6 +362,11 @@ class PackagesController < ApplicationController def query_packages packages = Package # .order(visits: :desc) + # Maps package id to the search strategy that put it into the + # result list: :exact, :name_prefix, :name_contains, :semantic or + # :fulltext. Empty when browsing without a search term. + @search_sources = {} + # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" @@ -393,15 +398,34 @@ class PackagesController < ApplicationController def search_packages(query) exact_match = Package.find_by(name: query) - return Package.where(id: exact_match.id) if exact_match + if exact_match + @search_sources[exact_match.id] = :exact + return Package.where(id: exact_match.id) + end - tokens = query.to_s.split(/\s+/) - name_matches = ( - Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) + - Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES) - ).uniq(&:id) + prefix_matches = Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES).to_a + token_matches = Package.name_contains_all(query.to_s.split(/\s+/)).limit( + MAX_PROMOTED_NAME_MATCHES + ).to_a + prefix_matches.each { |package| @search_sources[package.id] ||= :name_prefix } + token_matches.each { |package| @search_sources[package.id] ||= :name_contains } - results = (name_matches + semantic_results(query)).uniq(&:id) + semantic = [] + if Package.lexical_overlap?(query) + begin + semantic = Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a + semantic.each { |package| @search_sources[package.id] ||= :semantic } + rescue Vectorizer::Error => e + logger.error "Semantic search failed: #{e.message}" + flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' + semantic = Package.general_search(query).limit(100).to_a + semantic.each { |package| @search_sources[package.id] ||= :fulltext } + end + else + logger.debug "Query '#{query}' has no lexical overlap with package data" + end + + results = (prefix_matches + token_matches + semantic).uniq(&:id) # Queries that share no vocabulary with our package data can produce # no meaningful results. Skip the vector service round-trip and show @@ -409,23 +433,6 @@ class PackagesController < ApplicationController results.empty? ? Package.none : results end - # Semantic (vector) nearest neighbor search with a full-text fallback - # for when the external embedding service is unreachable. - def semantic_results(query) - unless Package.lexical_overlap?(query) - logger.debug "Query '#{query}' has no lexical overlap with package data" - return [] - end - - begin - Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a - rescue Vectorizer::Error => e - logger.error "Semantic search failed: #{e.message}" - flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' - Package.general_search(query).limit(100).to_a - end - end - # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index d7dec2c..3bc7de0 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -1,4 +1,16 @@ module PackagesHelper + # Human readable label for the search strategy that put a package + # into the search results (see PackagesController#search_packages) + def search_source_label(source) + { + exact: 'exact match', + name_prefix: 'name match', + name_contains: 'name match', + semantic: 'similar', + fulltext: 'full-text' + }[source] + end + # Return the description and/or the version of the package def screenshot_caption(screenshot) str = [] diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 53009da..2dfdedc 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -24,6 +24,9 @@ = pkg.name .text = pkg.description + - if @search_sources[pkg.id] + .text.search-source + span.label.secondary = search_source_label(@search_sources[pkg.id]) - elsif @view_style==:list - @packages.to_a.each do |pkg| .grid-x.grid-margin-x.listview @@ -37,6 +40,9 @@ a href=package_path(name: pkg.name) =pkg.name p =pkg.description + - if @search_sources[pkg.id] + p + span.label.secondary = search_source_label(@search_sources[pkg.id]) .listview.longdescription = pkg.long_description_first_paragraph diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index 92a6ea1..0c9c81c 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -28,6 +28,8 @@ class PackagesControllerTest < ActionController::TestCase get :grid, params: { search: 'vim' } assert_response :success assert_select 'div', 'vim' + assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources)) + assert_select '.search-source .label', 'exact match' end end @@ -39,6 +41,7 @@ class PackagesControllerTest < ActionController::TestCase # the grid renders every package in the database when no search # filter applies - so an empty result must not fall back to that assert_select 'div', text: /package-\d+/, count: 0 + assert_empty assigns(:search_sources) end end @@ -51,6 +54,9 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'package-0' # ...and before any semantically found package assert response.body.index('package-0') < response.body.index('vim') + sources = assigns(:search_sources) + assert_equal :name_prefix, sources[Package.find_by(name: 'package-0').id] + assert_equal :semantic, sources[packages(:vim).id] end end @@ -60,6 +66,8 @@ class PackagesControllerTest < ActionController::TestCase get :grid, params: { search: 'fire' } assert_response :success assert_select 'div', text: 'firefox', count: 1 + # the higher tier wins the attribution + assert_equal :name_prefix, assigns(:search_sources)[packages(:firefox).id] end end @@ -70,6 +78,9 @@ class PackagesControllerTest < ActionController::TestCase get :grid, params: { search: 'fire fox' } assert_response :success assert_select 'div', text: 'firefox', count: 1 + assert_equal({ packages(:firefox).id => :name_contains }, + assigns(:search_sources)) + assert_select '.search-source .label', 'name match' end end @@ -94,6 +105,7 @@ class PackagesControllerTest < ActionController::TestCase get :grid, params: { search: 'editor' } assert_response :success assert_select 'div', 'vim' + assert_equal({ packages(:vim).id => :semantic }, assigns(:search_sources)) end end @@ -109,6 +121,7 @@ class PackagesControllerTest < ActionController::TestCase assert_response :success assert_select 'div', 'vim' assert flash[:error].present? + assert_equal({ packages(:vim).id => :fulltext }, assigns(:search_sources)) end end From 4c736096aeb313d8abe6f46dd52352bdad6b543c Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 10:49:32 +0200 Subject: [PATCH 09/22] Show search source badges only with show_sources param The badges are a debugging aid - render them only when explicitly requested via ?show_sources=1 instead of on every search. --- app/helpers/packages_helper.rb | 6 ++++++ app/views/packages/browse.slim | 4 ++-- test/controllers/packages_controller_test.rb | 13 +++++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index 3bc7de0..b742a6b 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -1,4 +1,10 @@ module PackagesHelper + # Search result origin badges are only shown when explicitly + # requested via ?show_sources=1 + def search_sources_visible? + params[:show_sources].present? + end + # Human readable label for the search strategy that put a package # into the search results (see PackagesController#search_packages) def search_source_label(source) diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 2dfdedc..b8e09c9 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -24,7 +24,7 @@ = pkg.name .text = pkg.description - - if @search_sources[pkg.id] + - if search_sources_visible? && @search_sources[pkg.id] .text.search-source span.label.secondary = search_source_label(@search_sources[pkg.id]) - elsif @view_style==:list @@ -40,7 +40,7 @@ a href=package_path(name: pkg.name) =pkg.name p =pkg.description - - if @search_sources[pkg.id] + - if search_sources_visible? && @search_sources[pkg.id] p span.label.secondary = search_source_label(@search_sources[pkg.id]) .listview.longdescription diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index 0c9c81c..d72ad95 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -25,7 +25,7 @@ class PackagesControllerTest < ActionController::TestCase test 'exact package name search wins without calling the vector service' do with_stubbed_method(Package, :nearest_to_text, ->(_text) { flunk 'should not call vector search' }) do - get :grid, params: { search: 'vim' } + get :grid, params: { search: 'vim', show_sources: '1' } assert_response :success assert_select 'div', 'vim' assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources)) @@ -75,7 +75,7 @@ class PackagesControllerTest < ActionController::TestCase # "fire fox" should find "firefox" via name substring matching with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { Package.none }) do - get :grid, params: { search: 'fire fox' } + get :grid, params: { search: 'fire fox', show_sources: '1' } assert_response :success assert_select 'div', text: 'firefox', count: 1 assert_equal({ packages(:firefox).id => :name_contains }, @@ -84,6 +84,15 @@ class PackagesControllerTest < ActionController::TestCase end end + test 'search source badges stay hidden without show_sources param' do + with_stubbed_method(Package, :nearest_to_text, + ->(_text, **_opts) { Package.none }) do + get :grid, params: { search: 'fire fox' } + assert_response :success + assert_select '.search-source', count: 0 + end + end + test 'details shows semantically related packages' do packages(:vim).update_column(:embedding, Array.new(384, 0.25)) packages(:firefox).update_column(:embedding, Array.new(384, 0.75)) From 0aebb74b9b604d385bffd3504427227f66a28a06 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 10:51:37 +0200 Subject: [PATCH 10/22] Document search result sources and show_sources param --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 365d995..c330d9c 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,14 @@ Queries that share no vocabulary with the package data (gibberish, keyboard mash) return an honest empty result instead of random semantic matches. +**Why is this package in my results?** Each result knows which search +strategy produced it. The controller exposes this as +`@search_sources` (a hash mapping package ids to `:exact`, +`:name_prefix`, `:name_contains`, `:semantic` or `:fulltext`). For +debugging you can render a small badge on every result card by adding +`show_sources=1` to the URL, e.g. +`https://screenshots.debian.net/packages?search=sqlite&show_sources=1`. + Package pages also show semantically **related packages**, computed as a nearest neighbor query against the package's own description embedding - no external service call involved. From e3dff44f73cc3f7b2c39991f99d09d4dce373bf9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 11:03:05 +0200 Subject: [PATCH 11/22] Blacklist the libs section from package imports Libraries never produce useful screenshots. Excluding by Debian archive section instead of a lib* name pattern keeps applications with lib-prefixed names (libreoffice, librecad, ...) importable. With REMOVE_BLACKLISTED_PACKAGE set, the next import also removes previously imported libs-section packages and their screenshots. --- lib/tasks/import_debian.rake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 8f84233..666e074 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -29,6 +29,10 @@ BLACKLIST_SECTION_PATTERN = [ %r{/?oldlibs$}, %r{/?libdevel$}, %r{/?cli-mono$}, + # Libraries never make useful screenshots - and unlike a "lib*" name + # pattern this does not hit applications like libreoffice (editors) + # or librecad (graphics) + %r{/?libs$}, ] # Whether to delete a blacklisted package from the database From 3a1de72efe9fcfeb5b112688f45b06f4138ff814 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 11:20:36 +0200 Subject: [PATCH 12/22] Add temporary show=libs inspection view Lets you browse exactly the packages that the libs section blacklist would remove on the next import, to review them visually before running it. --- app/controllers/packages_controller.rb | 4 ++++ test/controllers/packages_controller_test.rb | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index eec639d..3be3c49 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -381,6 +381,10 @@ class PackagesController < ApplicationController when 'without' packages = packages.without_screenshots logger.debug 'Limiting packages to those without screenshots' + when 'libs' + # Temporary inspection view: exactly the packages that the libs + # section blacklist would remove on the next import + packages = packages.where("section ~* '/?(old)?libs$'") end packages diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index d72ad95..d18e81f 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -134,6 +134,16 @@ class PackagesControllerTest < ActionController::TestCase end end + test 'show=libs lists exactly the section-blacklist candidates' do + packages(:firefox).update_column(:section, 'libs') + packages(:vim).update_column(:section, 'editors') + + get :grid, params: { show: 'libs' } + assert_response :success + assert_select 'div', text: 'firefox', count: 1 + assert_select 'div', text: 'vim', count: 0 + end + # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') From a98398f7a1739c53f3a60564d6aceb7bae38eb35 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 12:33:28 +0200 Subject: [PATCH 13/22] Limit show=libs inspection view to packages with screenshots --- app/controllers/packages_controller.rb | 7 ++++--- test/controllers/packages_controller_test.rb | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 3be3c49..b86d1b5 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -382,9 +382,10 @@ class PackagesController < ApplicationController packages = packages.without_screenshots logger.debug 'Limiting packages to those without screenshots' when 'libs' - # Temporary inspection view: exactly the packages that the libs - # section blacklist would remove on the next import - packages = packages.where("section ~* '/?(old)?libs$'") + # Temporary inspection view: exactly the screenshot-bearing + # packages that the libs section blacklist would remove on the + # next import + packages = packages.where("section ~* '/?(old)?libs$'").with_screenshots end packages diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index d18e81f..ef6a7ef 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -134,9 +134,11 @@ class PackagesControllerTest < ActionController::TestCase end end - test 'show=libs lists exactly the section-blacklist candidates' do + test 'show=libs lists screenshot-bearing blacklist candidates only' do packages(:firefox).update_column(:section, 'libs') - packages(:vim).update_column(:section, 'editors') + # vim has no screenshots in the fixtures - even in libs section it + # must not appear + packages(:vim).update_column(:section, 'libs') get :grid, params: { show: 'libs' } assert_response :success From 07be9eadcc810c9650491a5fb76a6fc56df86d55 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 14:13:53 +0200 Subject: [PATCH 14/22] Remove the temporary show=libs inspection view The libs-section blacklist is confirmed safe; the review view has served its purpose. --- app/controllers/packages_controller.rb | 5 ----- test/controllers/packages_controller_test.rb | 12 ------------ 2 files changed, 17 deletions(-) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index b86d1b5..eec639d 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -381,11 +381,6 @@ class PackagesController < ApplicationController when 'without' packages = packages.without_screenshots logger.debug 'Limiting packages to those without screenshots' - when 'libs' - # Temporary inspection view: exactly the screenshot-bearing - # packages that the libs section blacklist would remove on the - # next import - packages = packages.where("section ~* '/?(old)?libs$'").with_screenshots end packages diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index ef6a7ef..d72ad95 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -134,18 +134,6 @@ class PackagesControllerTest < ActionController::TestCase end end - test 'show=libs lists screenshot-bearing blacklist candidates only' do - packages(:firefox).update_column(:section, 'libs') - # vim has no screenshots in the fixtures - even in libs section it - # must not appear - packages(:vim).update_column(:section, 'libs') - - get :grid, params: { show: 'libs' } - assert_response :success - assert_select 'div', text: 'firefox', count: 1 - assert_select 'div', text: 'vim', count: 0 - end - # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') From fd5747c6ab846ef7788a0874ac7c66997d0ecb7c Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sun, 23 Aug 2026 21:25:28 +0200 Subject: [PATCH 15/22] Deduplicate search results by package name If the database ever contains duplicate-name rows again, users should not see the same package twice in the results. --- app/controllers/packages_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index eec639d..602595b 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -425,7 +425,10 @@ class PackagesController < ApplicationController logger.debug "Query '#{query}' has no lexical overlap with package data" end - results = (prefix_matches + token_matches + semantic).uniq(&:id) + # Deduplicate by NAME, not id: if the database ever contains + # duplicate rows (see the packages_name_key incident) users should + # not see the same package twice. + results = (prefix_matches + token_matches + semantic).uniq(&:name) # Queries that share no vocabulary with our package data can produce # no meaningful results. Skip the vector service round-trip and show From 33fc169591b9dd816cd307618dfebc81e027e1aa Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 24 Aug 2026 22:11:05 +0200 Subject: [PATCH 16/22] Blacklist transitional/dummy stub packages from import Their short description reliably says so (e.g. 'transitional package', 'transitional dummy package for foo'). Verified against production package data: 190 matches, none of them ever had a screenshot. --- lib/tasks/import_debian.rake | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 666e074..c82d6a3 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -35,6 +35,14 @@ BLACKLIST_SECTION_PATTERN = [ %r{/?libs$}, ] +# List of regular expressions. If the package's short description +# matches any of these then the package will not be imported. +# Transitional/dummy stub packages exist only to ease upgrades to a +# renamed or split package and never get a useful screenshot. +BLACKLIST_DESCRIPTION_PATTERN = [ + /transitional/i, +] + # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true @@ -87,7 +95,9 @@ namespace :debshots do Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" - if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] + if package_name_blacklisted?(package[:Package]) or + package_section_blacklisted?(package[:Section]) or + package_description_blacklisted?(package[:Description]) # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] @@ -280,3 +290,16 @@ def package_section_blacklisted?(section) end return false end + +# Check if a package's short description marks it as a transitional or +# dummy stub package (e.g. "transitional package", "transitional dummy +# package for foo"). These never get a useful screenshot. +def package_description_blacklisted?(description) + BLACKLIST_DESCRIPTION_PATTERN.each do |pattern| + if description.to_s =~ pattern + Rails.logger.debug " > Blacklisted by description ('#{description}' matches '#{pattern}')" + return true + end + end + false +end From 8d26873baea087935a11d119095b96ac6daa0b49 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 24 Aug 2026 22:23:04 +0200 Subject: [PATCH 17/22] Widen transitional/dummy package description blacklist Add 'dummy package' and 'dependency package', the other phrases the Debian Developer's Reference (6.9.7) documents as convention and that deborphan --guess-dummy looks for. Deliberately skip bare 'dummy' and 'empty package': both have real false positives in production data (xserver-xorg-video-dummy, a bridge card 'double dummy solver' library, python3-roscreate's 'empty package template creator'). Verified against production data: 266 combined matches, still zero with screenshots. --- lib/tasks/import_debian.rake | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index c82d6a3..833bcb5 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -38,9 +38,17 @@ BLACKLIST_SECTION_PATTERN = [ # List of regular expressions. If the package's short description # matches any of these then the package will not be imported. # Transitional/dummy stub packages exist only to ease upgrades to a -# renamed or split package and never get a useful screenshot. +# renamed or split package and never get a useful screenshot. Debian +# has no dedicated metadata field for this - the Developer's Reference +# (6.9.7) documents these exact phrases as the convention package +# maintainers use, and as what deborphan --guess-dummy looks for. +# Deliberately not matching a bare "dummy": too many real packages use +# that word for actual functionality (e.g. xserver-xorg-video-dummy, +# fence-agents-dummy, "double dummy solver" bridge card libraries). BLACKLIST_DESCRIPTION_PATTERN = [ /transitional/i, + /dummy package/i, + /dependency package/i, ] # Whether to delete a blacklisted package from the database From 3f37e2d85743c8c4bb5fa009381f5faed035d32a Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 24 Aug 2026 22:47:03 +0200 Subject: [PATCH 18/22] Remove packages not found in any configured source Tracks every package name seen across all repositories/components/ architectures during update_from_deb_repos and, once the run completes, destroys database packages that were not among them (e.g. ones Debian has dropped entirely, or ones from a since-unconfigured suite/architecture). Deliberately opt-in (REMOVE_ORPHANED_PACKAGES = false by default), unlike REMOVE_BLACKLISTED_PACKAGE: a package looking 'orphaned' can also just mean a mirror had a transient fetch problem, so the removal refuses to run at all if any component/architecture failed to fetch, or if no packages were seen this run at all. --- lib/tasks/import_debian.rake | 74 ++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 833bcb5..f545d39 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -54,6 +54,15 @@ BLACKLIST_DESCRIPTION_PATTERN = [ # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true +# Whether to delete packages that were not found in any configured +# source during a full import run (e.g. packages Debian has dropped +# entirely, or ones from a suite/architecture no longer configured). +# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking +# "orphaned" can also mean a mirror had a transient fetch problem, so +# update_from_deb_repos refuses to act on it unless every configured +# component/architecture was actually fetched successfully this run. +REMOVE_ORPHANED_PACKAGES = false + namespace :debshots do desc "Import/update package database from configured DEB repositories" @@ -63,6 +72,13 @@ namespace :debshots do stats_updated = 0 stats_removed = 0 + # Every package name encountered in any configured source this run, + # used to find orphans afterwards. A Hash is used as a cheap set. + seen_package_names = {} + # Set to true if any component/architecture could not be fetched, + # so we know not to trust seen_package_names for orphan removal. + sources_incomplete = false + repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO @@ -94,6 +110,7 @@ namespace :debshots do packages = release.packages(component, architecture) unless packages Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" + sources_incomplete = true next end @@ -102,6 +119,7 @@ namespace :debshots do packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" + seen_package_names[package[:Package]] = true if package_name_blacklisted?(package[:Package]) or package_section_blacklisted?(package[:Section]) or @@ -160,6 +178,9 @@ namespace :debshots do Rails.logger.info "--------------------------------------------" end # repositories.each + stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete) + Rails.logger.info "#{stats_removed} packages removed in total" + end # task desc 'Update long description from i18n file' @@ -311,3 +332,56 @@ def package_description_blacklisted?(description) end false end + +# Remove packages that were not encountered in any configured source +# during this import run. Refuses to act if any component/architecture +# failed to fetch (sources_incomplete) or if seen_names came back empty +# (both would otherwise risk wiping out packages that are still very +# much alive in Debian, just missed because of a network hiccup). +# Returns the number of packages actually removed. +def orphan_removal_blocked?(seen_names, sources_incomplete) + if sources_incomplete + Rails.logger.warn ' > Skipping: not all configured components/architectures could be ' \ + 'fetched this run. Re-run when all sources are reachable.' + return true + end + + if seen_names.empty? + Rails.logger.warn ' > Skipping: no packages were seen this run at all - refusing to ' \ + 'treat that as "everything is orphaned".' + return true + end + + false +end + +def find_orphaned_package_ids(seen_names) + Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids| + ids << id unless seen_names.key?(name) + end +end + +def destroy_orphaned_packages(ids) + removed = 0 + Package.where(id: ids).find_each do |package| + Rails.logger.info " > Removing orphaned package '#{package.name}' from database" + package.destroy + removed += 1 + end + removed +end + +def remove_orphaned_packages(seen_names, sources_incomplete) + Rails.logger.info 'Checking for packages no longer found in any configured source' + return 0 if orphan_removal_blocked?(seen_names, sources_incomplete) + + orphan_ids = find_orphaned_package_ids(seen_names) + + unless REMOVE_ORPHANED_PACKAGES + Rails.logger.info " > #{orphan_ids.size} packages are orphaned (dry run - set " \ + 'REMOVE_ORPHANED_PACKAGES to actually remove them)' + return 0 + end + + destroy_orphaned_packages(orphan_ids) +end From c5cd77e9ee582ef866482c770924bf823b57ed08 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 4 Sep 2026 21:57:35 +0200 Subject: [PATCH 19/22] Stop concurrent imports from racing on package names Production imports kept aborting with ActiveRecord::RecordNotUnique on packages_name_key and - on environments that never got the constraint - silently created duplicate packages. Root cause: overlapping update_from_deb_repos runs (scheduled + manual) both do find-by-name then insert; whichever wins the race aborts the whole run, whichever loses gets a duplicate if the constraint was missing. Three things so the mechanism stops being the problem: - An advisory lock serializes update_from_deb_repos runs. A second run that finds the lock held simply skips with a warning instead of racing the first one. The lock is session-scoped, so it is released automatically when the process exits. - A per-package rescue for ActiveRecord::RecordNotUnique. If a create still races (e.g. a package being created by a web upload), the package is re-fetched and updated instead of aborting the run. - A real migration for packages_name_key, which previously only existed where it had been added by hand. From now on db:migrate creates it everywhere, new databases included. Also verified with a reproduction: a single sequential run of the deployed importer against a fresh database (both architectures, ~54k packages) creates zero duplicates. --- ..._add_unique_name_constraint_to_packages.rb | 28 ++++++++ lib/tasks/import_debian.rake | 64 +++++++++++++++---- 2 files changed, 78 insertions(+), 14 deletions(-) create mode 100644 db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb diff --git a/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb b/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb new file mode 100644 index 0000000..89efc03 --- /dev/null +++ b/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# Packages must be unique by name - the same package is listed once per +# architecture/component in the Debian archive and the importer looks it up +# by name before inserting, so anything that lets two rows for one name appear +# (e.g. concurrent import runs) breaks the site. Production had accidentally +# relied on a constraint that was only ever added ad-hoc on some databases +# and never tracked as a migration; this makes db:migrate the single source +# of truth. +class AddUniqueNameConstraintToPackages < ActiveRecord::Migration[8.1] + def up + add_unique_constraint :packages, :name, name: :packages_name_key unless constraint_exists? + end + + def down + remove_unique_constraint :packages, :name, name: :packages_name_key if constraint_exists? + end + + private + + def constraint_exists? + select_value(<<~SQL.squish) + SELECT 1 + FROM pg_constraint + WHERE conrelid = 'packages'::regclass AND conname = 'packages_name_key' AND contype = 'u' + SQL + end +end \ No newline at end of file diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index f545d39..5569c07 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -54,6 +54,13 @@ BLACKLIST_DESCRIPTION_PATTERN = [ # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true +# Postgres advisory lock used to serialize update_from_deb_repos runs. +# Without this, a scheduled and a manual run (or two scheduled runs) can +# overlap: both do find_by+insert for the same name, and the second one +# either violates packages_name_key (aborting the whole import) or - on +# instances that never had that constraint - silently creates a duplicate. +IMPORT_ADVISORY_LOCK_KEY = 0x0DE65507 + # Whether to delete packages that were not found in any configured # source during a full import run (e.g. packages Debian has dropped # entirely, or ones from a suite/architecture no longer configured). @@ -84,6 +91,11 @@ namespace :debshots do Rails.logger.level = Logger::INFO #Rails.logger.level = Logger::DEBUG + unless import_lock_acquired? + Rails.logger.warn 'Another import run already in progress (advisory lock). Skipping.' + exit 0 + end + Rails.logger.info "Importing Debian package information" repositories.each do |repository| @@ -137,24 +149,37 @@ namespace :debshots do next end # if blacklisted - db_package = Package.find_by name: package[:Package] - if db_package # package exists in the database - # check if the parsed package data has a newer version - current_version = DebImporter::Version.new(db_package.version) - new_version = DebImporter::Version.new(package[:Version]) + begin + db_package = Package.find_by name: package[:Package] + if db_package # package exists in the database + # check if the parsed package data has a newer version + current_version = DebImporter::Version.new(db_package.version) + new_version = DebImporter::Version.new(package[:Version]) - Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" - if new_version > current_version - # update the package information from new data + Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" + if new_version > current_version + # update the package information from new data + update_data(package, db_package) + Rails.logger.info "Updating package information" + stats_updated += 1 + end + else # new package - create it in the database + Rails.logger.debug "No such package in our database. Creating one." + db_package = Package.new + update_data(package, db_package) + stats_added += 1 + end + rescue ActiveRecord::RecordNotUnique + # Lost a create-race: another process inserted this package name + # between our find and our insert (a web upload, or an import on + # an instance without the advisory lock). Treat it as found and + # update instead of aborting the whole run. + Rails.logger.warn "Package '#{package[:Package]}' was created " \ + 'concurrently. Updating it instead.' + if (db_package = Package.find_by(name: package[:Package])) update_data(package, db_package) - Rails.logger.info "Updating package information" stats_updated += 1 end - else # new package - create it in the database - Rails.logger.debug "No such package in our database. Creating one." - db_package = Package.new - update_data(package, db_package) - stats_added += 1 end # unless db_package @@ -385,3 +410,14 @@ def remove_orphaned_packages(seen_names, sources_incomplete) destroy_orphaned_packages(orphan_ids) end + +# Try to take the advisory lock guarding update_from_deb_repos. The lock lives +# on this database connection/session, so it is released automatically when the +# process exits - no explicit unlock needed. Returns true if this run is the +# one that may proceed. +def import_lock_acquired? + result = ActiveRecord::Base.connection.execute( + "SELECT pg_try_advisory_lock(#{IMPORT_ADVISORY_LOCK_KEY})" + ) + result.first['pg_try_advisory_lock'] == true +end From 7fff0a6dbf6b4939a0ac7d8f068ed1c8c306dc52 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 4 Sep 2026 21:57:48 +0200 Subject: [PATCH 20/22] Bump schema version for constraint migration --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index a2ad488..433a8e2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_08_21_210000) do +ActiveRecord::Schema[8.1].define(version: 2026_09_04_120000) do # These are extensions that must be enabled in order to support this database enable_extension "pg_catalog.plpgsql" enable_extension "vector" From 81b87f2cc37bb14277137ba7df1fc3aa66b18ca9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sat, 5 Sep 2026 13:15:29 +0200 Subject: [PATCH 21/22] Record git revision in build and update deploy Makefile --- Dockerfile | 8 ++++++-- Makefile | 11 ++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91a0c7c..e451fa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ARG RUBY_VERSION=3.4.7 ARG NODE_VERSION=22.16.0 ARG BUN_VERSION=1.3.9 +ARG GIT_REV=unknown # --- Stage 1: Grab Node.js Binaries --- # We use the official image as a source for pre-compiled binaries @@ -78,8 +79,11 @@ RUN groupadd --system --gid 1000 rails && \ useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \ chown -R rails:rails db log tmp public -# Create versioninfo with build timestamp -RUN date +"%Y-%m-%d %H:%M:%S %Z" > public/versioninfo +# Redeclare so it is in scope in this final stage +ARG GIT_REV=unknown + +# Create versioninfo with git revision and build timestamp +RUN printf 'git-rev: %s\nbuilt-at: %s\n' "$GIT_REV" "$(date +'%Y-%m-%d %H:%M:%S %Z')" > public/versioninfo USER 1000:1000 diff --git a/Makefile b/Makefile index b83b2e3..9c165cd 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ -include .env.production .env.development -REGISTRY := registry.coolify1.workaround.org +REGISTRY := git.workaround.org/chaas -stage: TAG=stage +stage: TAG=latest stage: build tag push curl -s -H "Authorization: Bearer $(COOLIFY_STAGE_DEPLOY_TOKEN)" \ -X POST "$(COOLIFY_STAGE_DEPLOY_WEBHOOK)" @@ -15,10 +15,11 @@ prod: build tag push -X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)" build: - podman build -t debshots-$(TAG) . + docker build --build-arg GIT_REV=$(git rev-parse --short HEAD) -t debshots-$(TAG) . tag: - podman tag debshots-$(TAG) $(REGISTRY)/debshots-web:$(TAG) + docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG) push: - podman push $(REGISTRY)/debshots-web:$(TAG) + docker push $(REGISTRY)/debshots:$(TAG) + From 51392f985404980f30285e7d604bea7c4bf1d1c9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Sat, 5 Sep 2026 13:41:11 +0200 Subject: [PATCH 22/22] Fix GIT_REV build arg expansion in deploy Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9c165cd..f2799a9 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ prod: build tag push -X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)" build: - docker build --build-arg GIT_REV=$(git rev-parse --short HEAD) -t debshots-$(TAG) . + docker build --build-arg GIT_REV=$(shell git rev-parse --short HEAD) -t debshots-$(TAG) . tag: docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG)