diff --git a/Dockerfile b/Dockerfile index 91a0c7c..177771d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ARG BUN_VERSION=1.3.9 # --- Stage 1: Grab Node.js Binaries --- # We use the official image as a source for pre-compiled binaries -FROM docker.io/library/node:${NODE_VERSION} AS node_source +FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source # --- Stage 2: Base (Runtime Environment) --- FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base @@ -36,13 +36,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ apt-get install --no-install-recommends -y \ - build-essential git libyaml-dev pkg-config python-is-python3 + build-essential git libyaml-dev node-gyp pkg-config python-is-python3 -# Install Node.js and npm from node_source -COPY --from=node_source /usr/local/ /usr/local/ +# "Heist" Node.js and npm from the node_source stage (build-time only) +COPY --from=node_source /usr/local/bin/node /usr/local/bin/node +COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules -# Install Bun -RUN npm install -g bun@${BUN_VERSION} +# Re-link npm and install Bun (build-time only) +RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ + npm install -g bun@${BUN_VERSION} # 1. Install Gems (Separate COPY for maximum caching) COPY Gemfile Gemfile.lock ./ diff --git a/Gemfile b/Gemfile index 6e1b32d..c1e1ea4 100644 --- a/Gemfile +++ b/Gemfile @@ -41,7 +41,8 @@ gem 'mini_magick' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false - +# Structured logging for production +gem 'lograge' # Deploy this application anywhere as a Docker container [https://kamal-deploy.org] gem 'kamal', require: false @@ -123,10 +124,6 @@ gem 'will_paginate-foundation' # Full-text search in PostgreSQL gem 'pg_search' -# Nearest neighbor search for PostgreSQL using the pgvector extension -# https://github.com/ankane/neighbor -gem 'neighbor' - # Use SLIM as our templating language gem 'slim-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 9cf75ac..202b1e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -224,6 +224,11 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.7.0) + lograge (0.14.0) + actionpack (>= 4) + activesupport (>= 4) + railties (>= 4) + request_store (~> 1.0) loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -250,8 +255,6 @@ GEM minitest (>= 5.0, < 7) ruby-progressbar msgpack (1.8.0) - neighbor (1.2.0) - activerecord (>= 7.2) nenv (0.3.0) net-http (0.9.1) uri (>= 0.11.1) @@ -424,6 +427,8 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) + request_store (1.7.0) + rack (>= 1.4) responders (3.2.0) actionpack (>= 7.0) railties (>= 7.0) @@ -580,10 +585,10 @@ DEPENDENCIES jbuilder kamal listen (~> 3.5) + lograge mini_magick minitest-rails minitest-reporters - neighbor omniauth omniauth-rails_csrf_protection omniauth_openid_connect diff --git a/README.md b/README.md index deaef3a..55facc4 100644 --- a/README.md +++ b/README.md @@ -15,30 +15,6 @@ https://salsa.debian.org/debian/debshots/-/issues Check out the doc/README.Development.md -## Package search - -The package search combines several strategies, tried in this order: - -1. **Exact package name**: Searching for `vim` shows exactly this - package without calling any external service. -2. **Name prefix matches**: Packages whose name starts with the query - rank first, e.g. searching for `sqlite` promotes `sqlite3`, - `sqlitebrowser` and friends above all other results. -3. **Compound word splitting**: Packages whose name contains *all* - query tokens match too, so `sqlite browser` finds `sqlitebrowser`. -4. **Semantic search**: The query is embedded via the vector service - (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among - the package description embeddings are returned, using the pgvector - extension with an HNSW cosine index. Embeddings are computed from - the package description and long description and can be (re-)built - with `bin/rails debshots:compute_vectors`. -5. **Full-text fallback**: If the vector service cannot be reached, - classic PostgreSQL full-text search takes over. - -Queries that share no vocabulary with the package data (gibberish, -keyboard mash) return an honest empty result instead of random -semantic matches. - ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 726f4f0..b242467 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -362,7 +362,7 @@ class PackagesController < ApplicationController # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" - packages = search_packages(params[:search]) + packages = packages.general_search(params[:search]) end case params[:show] @@ -378,51 +378,6 @@ class PackagesController < ApplicationController packages end - # Text search for packages: an exact package name match wins without - # touching the vector service. Otherwise packages whose name starts - # with the query (or contains all query tokens, catching compound - # words like "sqlite browser" -> "sqlitebrowser") are promoted above - # the semantic (vector) nearest neighbor results. Classic PostgreSQL - # full-text search kicks in when the vector service cannot be reached. - MAX_PROMOTED_NAME_MATCHES = 8 - MAX_SEMANTIC_RESULTS = 100 - private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS - - def search_packages(query) - exact_match = Package.find_by(name: query) - return Package.where(id: exact_match.id) if exact_match - - tokens = query.to_s.split(/\s+/) - name_matches = ( - Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES) + - Package.name_contains_all(tokens).limit(MAX_PROMOTED_NAME_MATCHES) - ).uniq(&:id) - - results = (name_matches + semantic_results(query)).uniq(&:id) - - # Queries that share no vocabulary with our package data can produce - # no meaningful results. Skip the vector service round-trip and show - # an honest empty result instead of random packages. - results.empty? ? Package.none : results - end - - # Semantic (vector) nearest neighbor search with a full-text fallback - # for when the external embedding service is unreachable. - def semantic_results(query) - unless Package.lexical_overlap?(query) - logger.debug "Query '#{query}' has no lexical overlap with package data" - return [] - end - - begin - Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a - rescue Vectorizer::Error => e - logger.error "Semantic search failed: #{e.message}" - flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' - Package.general_search(query).limit(100).to_a - end - end - # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/models/package.rb b/app/models/package.rb index e2e1b00..441f5c4 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -2,9 +2,6 @@ require 'open-uri' # allows to load URLs using open() require 'json' require 'deb_importer' -# Search and embedding logic keeps accumulating here; the class is -# cohesive enough that splitting it is not worth the indirection. -# rubocop:disable Metrics/ClassLength class Package < ApplicationRecord # PostgreSQL-based full-text search: # https://github.com/Casecommons/pg_search @@ -29,11 +26,6 @@ class Package < ApplicationRecord inverse_of: :package, dependent: :destroy - # Nearest neighbor search using pgvector. The "embedding" column stores - # a vector computed from description + long_description by an external - # web service (see Vectorizer and lib/tasks/vectorize_packages.rake). - has_neighbors :embedding - # default_scope { # order('name ASC') # } @@ -48,94 +40,6 @@ class Package < ApplicationRecord long_description.split(/\n\.\n/).first if long_description end - # The text that the embedding vector is computed from: description - # and long description combined. - def embedding_text - [description, long_description].compact.join("\n") - end - - # Compute and store the embedding vector for this package by asking - # the external vector service. Raises Vectorizer::Error on failure. - # Returns true if the vector was saved. - def update_embedding! - text = embedding_text - return false if text.blank? - - self.embedding = Vectorizer.embed(text) - save! - end - - # Return a relation of packages whose stored embeddings are closest to - # the given query vector (an array of floats). Records get a - # "neighbor_distance" attribute added (cosine distance, lower is better). - # The returned relation can still be chained (e.g. filtered or paginated). - # - # Only returns packages that already have an embedding computed. - def self.nearest_to_vector(vector, limit: nil) - neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine') - limit ? neighbors.limit(limit) : neighbors - end - - # Turn a free-text search string into a vector via the external web - # service and find the closest packages. - def self.nearest_to_text(text, limit: nil) - return none if text.blank? - - nearest_to_vector(Vectorizer.embed(text), limit: limit) - end - - # Packages whose name starts with the given string - used to promote - # obvious matches like "sqlite3" for a "sqlite" search. - def self.name_starts_with(text) - return none if text.blank? - - where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name) - end - - # Packages whose name contains all given tokens as substrings. Catches - # compound words split by the user, e.g. "sqlite browser" finding the - # "sqlitebrowser" package. - def self.name_contains_all(tokens) - tokens = tokens.select { |token| token.length >= 2 } - return none if tokens.empty? - - scope = self - tokens.each do |token| - scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\') - end - scope.order(:name) - end - - def self.escape_for_like(text) - # Escape LIKE wildcards so that package name characters like "+" or "." - # cannot break the pattern - text.gsub(/[\\%_]/) { |char| "\\#{char}" } - end - - # Check whether the given text shares at least one lexeme with any - # package name/description in the database. Queries without such an - # overlap (gibberish, keyboard mash, unsupported languages) can - # neither match full-text nor produce meaningful semantic results - - # the embedding model maps unknown tokens close to the corpus - # centroid, making them look like good matches. - def self.lexical_overlap?(text) - tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) } - return false if tokens.empty? - - # Pin the same text search dictionary that the packages_fts index - # uses - the server-wide default can be anything. - tsquery = tokens.map do |token| - sanitize_sql(["plainto_tsquery('english', ?)", token]) - end.join(' || ') - # The expression must match the packages_fts GIN index definition - # so that Postgres can use the index. - where( - "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \ - "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \ - "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})" - ).exists? - end - # Return a query of all packages that have screenshots def self.with_screenshots # Query for all packages who's ID appears in a screenshot's "package_id" field diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index 53009da..84d8599 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -14,7 +14,7 @@ - if @packages.any? - if @view_style==:grid .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium" - - @packages.each do |pkg| + - @packages.all.each do |pkg| .cell.pkgcard data-equalizer-watch=true a.black href=package_path(name: pkg.name) .image diff --git a/config/environments/production.rb b/config/environments/production.rb index b1c0f32..ced9b3c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -37,6 +37,15 @@ Rails.application.configure do config.log_tags = [:request_id] config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) + # Lograge - structured request logging + config.lograge.enabled = true + config.lograge.custom_options = lambda do |event| + { + request_id: event.payload[:request_id], + user_id: event.payload[:user_id] + } + end + # Change to "debug" to log everything (including potentially personally-identifiable information!). config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info') diff --git a/config/initializers/will_paginate.rb b/config/initializers/will_paginate.rb deleted file mode 100644 index e36b7b3..0000000 --- a/config/initializers/will_paginate.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -# Enable will_paginate's Array#paginate so that merged search results -# (name prefix matches + semantic results) can be paginated like -# ActiveRecord relations. -require 'will_paginate/array' diff --git a/db/migrate/20260821203833_install_neighbor_vector.rb b/db/migrate/20260821203833_install_neighbor_vector.rb deleted file mode 100644 index d591f1c..0000000 --- a/db/migrate/20260821203833_install_neighbor_vector.rb +++ /dev/null @@ -1,5 +0,0 @@ -class InstallNeighborVector < ActiveRecord::Migration[8.1] - def change - enable_extension "vector" - end -end diff --git a/db/migrate/20260821210000_add_embedding_to_packages.rb b/db/migrate/20260821210000_add_embedding_to_packages.rb deleted file mode 100644 index 62825f9..0000000 --- a/db/migrate/20260821210000_add_embedding_to_packages.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -# Adds a vector column to store text embeddings of the package -# description and long description. The vectors are computed by an -# external web service and used for nearest neighbor search. -# -# The pgvector extension is enabled by the InstallNeighborVector migration. -class AddEmbeddingToPackages < ActiveRecord::Migration[8.1] - def change - add_column :packages, :embedding, :vector, limit: 384 - - # HNSW index for fast approximate nearest neighbor searches - add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops - end -end diff --git a/db/schema.rb b/db/schema.rb index a2ad488..2c57b69 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,131 +10,133 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_08_21_210000) do - # These are extensions that must be enabled in order to support this database - enable_extension "pg_catalog.plpgsql" - enable_extension "vector" +ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do + # These are extensions that must be enabled to support this database + enable_extension 'pg_catalog.plpgsql' - create_table "action_mailbox_inbound_emails", force: :cascade do |t| - t.datetime "created_at", null: false - t.string "message_checksum", null: false - t.string "message_id", null: false - t.integer "status", default: 0, null: false - t.datetime "updated_at", null: false - t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true + create_table 'action_mailbox_inbound_emails', force: :cascade do |t| + t.integer 'status', default: 0, null: false + t.string 'message_id', null: false + t.string 'message_checksum', null: false + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['message_id', 'message_checksum'], +name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true end - create_table "action_text_rich_texts", force: :cascade do |t| - t.text "body" - t.datetime "created_at", null: false - t.string "name", null: false - t.bigint "record_id", null: false - t.string "record_type", null: false - t.datetime "updated_at", null: false - t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true + create_table 'action_text_rich_texts', force: :cascade do |t| + t.string 'name', null: false + t.text 'body' + t.string 'record_type', null: false + t.bigint 'record_id', null: false + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', +unique: true end - create_table "active_storage_attachments", force: :cascade do |t| - t.bigint "blob_id", null: false - t.datetime "created_at", precision: nil, null: false - t.string "name", null: false - t.bigint "record_id", null: false - t.string "record_type", null: false - t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" - t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + create_table 'active_storage_attachments', force: :cascade do |t| + t.string 'name', null: false + t.string 'record_type', null: false + t.bigint 'record_id', null: false + t.bigint 'blob_id', null: false + t.datetime 'created_at', precision: nil, null: false + t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id' + t.index ['record_type', 'record_id', 'name', 'blob_id'], +name: 'index_active_storage_attachments_uniqueness', unique: true end - create_table "active_storage_blobs", force: :cascade do |t| - t.bigint "byte_size", null: false - t.string "checksum" - t.string "content_type" - t.datetime "created_at", precision: nil, null: false - t.string "filename", null: false - t.string "key", null: false - t.text "metadata" - t.string "service_name", null: false - t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + create_table 'active_storage_blobs', force: :cascade do |t| + t.string 'key', null: false + t.string 'filename', null: false + t.string 'content_type' + t.text 'metadata' + t.bigint 'byte_size', null: false + t.string 'checksum' + t.datetime 'created_at', precision: nil, null: false + t.string 'service_name', null: false + t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true end - create_table "active_storage_variant_records", force: :cascade do |t| - t.bigint "blob_id", null: false - t.string "variation_digest", null: false - t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + create_table 'active_storage_variant_records', force: :cascade do |t| + t.bigint 'blob_id', null: false + t.string 'variation_digest', null: false + t.index ['blob_id', 'variation_digest'], +name: 'index_active_storage_variant_records_uniqueness', unique: true end - create_table "logs", id: :serial, force: :cascade do |t| - t.datetime "created_at", precision: nil, null: false - t.inet "ip_address" - t.string "level" - t.string "message" - t.integer "package_id" - t.integer "screenshot_id" - t.string "section" - t.datetime "updated_at", precision: nil, null: false - t.integer "user_id" + create_table 'logs', id: :serial, force: :cascade do |t| + t.string 'message' + t.string 'level' + t.string 'section' + t.inet 'ip_address' + t.datetime 'created_at', precision: nil, null: false + t.datetime 'updated_at', precision: nil, null: false + t.integer 'user_id' + t.integer 'package_id' + t.integer 'screenshot_id' end - create_table "packages", id: :serial, force: :cascade do |t| - t.datetime "created_at", precision: nil - t.string "description", limit: 80 - t.vector "embedding", limit: 384 - t.string "homepage", limit: 400 - t.text "long_description" - t.string "maintainer", limit: 100 - t.string "maintainer_email", limit: 100 - t.string "name", null: false - t.string "origin", limit: 80 - t.string "section", limit: 50 - t.datetime "updated_at", precision: nil - t.string "version", limit: 200 - t.integer "visits", default: 0 - t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin - t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw - t.unique_constraint ["name"], name: "packages_name_key" + create_table 'packages', id: :serial, force: :cascade do |t| + t.string 'name', null: false + t.string 'description', limit: 80 + t.string 'section', limit: 50 + t.string 'maintainer', limit: 100 + t.string 'maintainer_email', limit: 100 + t.string 'homepage', limit: 400 + t.string 'version', limit: 200 + t.text 'long_description' + t.string 'origin', limit: 80 + t.datetime 'created_at', precision: nil + t.datetime 'updated_at', precision: nil + t.integer 'visits', default: 0 + t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", +name: 'packages_fts', using: :gin + t.unique_constraint ['name'], name: 'packages_name_key' end - create_table "screenshots", id: :serial, force: :cascade do |t| - t.boolean "approved", default: false, null: false - t.datetime "created_at", precision: nil - t.text "description" - t.boolean "hidden", default: false - t.string "image_fingerprint" - t.integer "package_id" - t.text "simage_data" - t.datetime "updated_at", precision: nil - t.string "uploaderhash", limit: 72 - t.inet "uploaderip" - t.integer "user_id", default: 0 - t.string "version", limit: 50 - t.index ["id", "approved"], name: "id_approved" - t.index ["id", "uploaderhash"], name: "id_uploaderhash" + create_table 'screenshots', id: :serial, force: :cascade do |t| + t.integer 'package_id' + t.string 'version', limit: 50 + t.datetime 'created_at', precision: nil + t.string 'uploaderhash', limit: 72 + t.boolean 'approved', default: false, null: false + t.text 'description' + t.datetime 'updated_at', precision: nil + t.string 'image_fingerprint' + t.integer 'user_id', default: 0 + t.text 'simage_data' + t.inet 'uploaderip' + t.boolean 'hidden', default: false + t.index ['id', 'approved'], name: 'id_approved' + t.index ['id', 'uploaderhash'], name: 'id_uploaderhash' end - create_table "users", id: :serial, force: :cascade do |t| - t.boolean "admin_role", default: false - t.integer "approved_screenshots", default: 0 - t.datetime "created_at", precision: nil, null: false - t.datetime "current_sign_in_at", precision: nil - t.inet "current_sign_in_ip" - t.string "email", default: "", null: false - t.string "encrypted_password", default: "", null: false - t.integer "failed_attempts", default: 0, null: false - t.datetime "last_sign_in_at", precision: nil - t.inet "last_sign_in_ip" - t.datetime "locked_at", precision: nil - t.boolean "moderator_role", default: false - t.text "name" - t.string "provider" - t.boolean "pseudo", default: false - t.integer "rejected_screenshots", default: 0 - t.integer "sign_in_count", default: 0, null: false - t.string "uid" - t.string "unlock_token" - t.datetime "updated_at", precision: nil, null: false - t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true + create_table 'users', id: :serial, force: :cascade do |t| + t.text 'name' + t.datetime 'created_at', precision: nil, null: false + t.datetime 'updated_at', precision: nil, null: false + t.string 'email', default: '', null: false + t.string 'encrypted_password', default: '', null: false + t.integer 'sign_in_count', default: 0, null: false + t.datetime 'current_sign_in_at', precision: nil + t.datetime 'last_sign_in_at', precision: nil + t.inet 'current_sign_in_ip' + t.inet 'last_sign_in_ip' + t.integer 'failed_attempts', default: 0, null: false + t.string 'unlock_token' + t.datetime 'locked_at', precision: nil + t.string 'provider' + t.string 'uid' + t.boolean 'admin_role', default: false + t.boolean 'moderator_role', default: false + t.boolean 'pseudo', default: false + t.integer 'approved_screenshots', default: 0 + t.integer 'rejected_screenshots', default: 0 + t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true end - add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" - add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey" + add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id' + add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id' + add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey' end diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake deleted file mode 100644 index 1c19837..0000000 --- a/lib/tasks/vectorize_packages.rake +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -namespace :debshots do - desc 'Compute embedding vectors for all packages based on their description ' \ - 'and long description (missing ones only unless FORCE=1). ' \ - 'Set CONCURRENCY=n to control parallel HTTP requests.' - task compute_vectors: :environment do - logger = Logger.new($stdout) - logger.level = Logger::INFO - - force = ENV['FORCE'].present? - concurrency = [ENV.fetch('CONCURRENCY', '4').to_i, 1].max - - # Never use more threads than the connection pool has connections. - # Keep one connection reserved for the main thread. - pool_size = ActiveRecord::Base.connection_pool.size - concurrency = [concurrency, pool_size - 1].min - concurrency = 0 if concurrency.negative? - - scope = force ? Package.all : Package.where(embedding: nil) - total = scope.count - - if total.zero? - logger.info 'No packages need vector computation.' - next - end - - logger.info "Computing vectors for #{total} packages " \ - "(#{force ? 'recomputing' : 'missing only'}, " \ - "service: #{Vectorizer.service_url}, " \ - "concurrency: #{concurrency.zero? ? 1 : concurrency})" - - stats_mutex = Mutex.new - stats = { done: 0, failed: 0 } - - process_package = lambda do |package| - stats_mutex.synchronize do - begin - raise ArgumentError, 'no description text' if package.embedding_text.blank? - - package.update_embedding! - stats[:done] += 1 - rescue Vectorizer::Error, ArgumentError => e - stats[:failed] += 1 - logger.error "Failed for package #{package.name}: #{e.message}" - end - - processed = stats[:done] + stats[:failed] - return unless (processed % 100).zero? - - percent = (processed.to_f / total * 100).round(1) - logger.info "Progress: #{processed}/#{total} (#{percent}%)" - end - end - - if concurrency.zero? - # Sequential fallback when the connection pool has no room for workers - scope.find_each { |package| process_package.call(package) } - else - # Feed package IDs into a queue that the worker threads pull from. - work_queue = Queue.new - scope.pluck(:id).each { |id| work_queue << id } - concurrency.times { work_queue << nil } # one poison pill per worker - - workers = Array.new(concurrency) do - Thread.new do - # Each thread gets its own database connection from the pool. - ActiveRecord::Base.connection_pool.with_connection do - while (package_id = work_queue.pop) - package = Package.find_by(id: package_id) - process_package.call(package) if package - end - end - rescue StandardError => e - logger.error "Worker crashed: #{e.message}" - end - end - - workers.each(&:join) - end - - logger.info "Done. #{stats[:done]} vectors computed, #{stats[:failed]} failures." - end -end diff --git a/lib/vectorizer.rb b/lib/vectorizer.rb deleted file mode 100644 index 84df3c4..0000000 --- a/lib/vectorizer.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'json' - -# Client for the external web service that turns text into embedding -# vectors for nearest neighbor search (pgvector). -# -# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...] -# -# The service URL can be overridden with the VECTOR_SERVICE_URL -# environment variable. -class Vectorizer - # Raised when the vector service cannot be reached or returns garbage - class Error < StandardError; end - - DEFAULT_URL = 'https://vector-search.workaround.org/vector' - - class << self - # Compute the embedding vector for the given text. - # Returns an array of floats. - def embed(text) - new.embed(text) - end - - def service_url - ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL) - end - end - - def embed(text) - response = post_text(text) - - unless response.is_a?(Net::HTTPSuccess) - raise Error, "Vector service returned HTTP #{response.code}" - end - - JSON.parse(response.body).fetch('vector') - rescue JSON::ParserError => e - raise Error, "Vector service returned invalid JSON: #{e.message}" - rescue KeyError - raise Error, 'Vector service response did not contain a "vector" field' - end - - private - - def post_text(text) - uri = URI.parse(self.class.service_url) - request = build_request(uri, text) - - options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 } - Net::HTTP.start(uri.host, uri.port, **options) do |http| - http.request(request) - end - rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e - raise Error, "Could not reach vector service: #{e.message}" - end - - def build_request(uri, text) - request = Net::HTTP::Post.new(uri.request_uri) - request['Content-Type'] = 'application/json' - request.body = { text: text }.to_json - - # Optional Bearer token if the service runs with API_SECRET set - secret = ENV['VECTOR_SERVICE_SECRET'] - request['Authorization'] = "Bearer #{secret}" if secret.present? - - request - end -end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index 4d89d88..c9eae31 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require 'test_helper' class PackagesControllerTest < ActionController::TestCase @@ -22,81 +20,6 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'A program to browse web sites.' end - test 'exact package name search wins without calling the vector service' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text) { flunk 'should not call vector search' }) do - get :grid, params: { search: 'vim' } - assert_response :success - assert_select 'div', 'vim' - end - end - - test 'gibberish search shows no results and never calls the vector service' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text) { flunk 'should not call vector search' }) do - get :grid, params: { search: 'asdfgh zzzz' } - assert_response :success - # the grid renders every package in the database when no search - # filter applies - so an empty result must not fall back to that - assert_select 'div', text: /package-\d+/, count: 0 - end - end - - test 'name prefix matches rank above semantic results' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do - get :grid, params: { search: 'package' } - assert_response :success - # fixtures create package-0..package-99 - prefix hits must render - assert_select 'div', 'package-0' - # ...and before any semantically found package - assert response.body.index('package-0') < response.body.index('vim') - end - end - - test 'prefix matches and semantic results are deduplicated' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do - get :grid, params: { search: 'fire' } - assert_response :success - assert_select 'div', text: 'firefox', count: 1 - end - end - - test 'compound words in search match packages with joined names' do - # "fire fox" should find "firefox" via name substring matching - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.none }) do - get :grid, params: { search: 'fire fox' } - assert_response :success - assert_select 'div', text: 'firefox', count: 1 - end - end - - test 'search uses semantic (vector) results' do - relation = Package.where(name: packages(:vim).name) - with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do - get :grid, params: { search: 'editor' } - assert_response :success - assert_select 'div', 'vim' - end - end - - # There is deliberately no fallback for an empty vector search - # result: KNN always returns something as long as packages have - # embeddings. Full-text search is only used when the vector service - # cannot be reached. - - test 'search falls back to full-text search when vector service fails' do - with_stubbed_method(Vectorizer, :embed, - ->(_text) { raise Vectorizer::Error, 'service down' }) do - get :grid, params: { search: 'editor' } - assert_response :success - assert_select 'div', 'vim' - assert flash[:error].present? - end - end - # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') diff --git a/test/models/package_test.rb b/test/models/package_test.rb index 36562a2..124f6c4 100644 --- a/test/models/package_test.rb +++ b/test/models/package_test.rb @@ -1,107 +1,7 @@ -# frozen_string_literal: true - require 'test_helper' class PackageTest < ActiveSupport::TestCase - test 'embedding_text combines description and long description' do - package = packages(:firefox) - assert_equal "A web browser\nA program to browse web sites.", package.embedding_text - end - - test 'embedding_text works with missing long description' do - package = packages(:firefox) - package.long_description = nil - assert_equal 'A web browser', package.embedding_text - end - - test 'update_embedding! stores the vector returned by the vector service' do - vector = Array.new(384) { |i| i / 384.0 } - with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do - package = packages(:vim) - assert package.update_embedding! - # The vector type stores float32 so allow small rounding differences - package.reload.embedding.to_a.each_with_index do |actual, i| - assert_in_delta vector[i], actual, 1e-6 - end - end - end - - test 'update_embedding! does nothing without description text' do - package = packages(:firefox) - package.description = nil - package.long_description = nil - with_stubbed_method(Vectorizer, :embed, - ->(_text) { flunk 'should not call the vector service' }) do - assert_not package.update_embedding! - end - end - - test 'nearest_to_text returns none for blank text' do - assert_empty Package.nearest_to_text('') - assert_empty Package.nearest_to_text(nil) - end - - test 'name_starts_with matches name prefixes case-insensitively' do - assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name - assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name - assert_empty Package.name_starts_with('') - assert_empty Package.name_starts_with(nil) - end - - test 'name_contains_all finds compound words split by spaces' do - assert_includes Package.name_contains_all(%w[fire fox]).map(&:name), - packages(:firefox).name - # all tokens must be present in the name - assert_empty Package.name_contains_all(%w[firefox zzzz]) - assert_empty Package.name_contains_all([]) - # single character tokens would flood results - ignore them - assert_empty Package.name_contains_all(['f']) - end - - test 'lexical_overlap? pins the english dictionary regardless of server config' do - # Regression: with default_text_search_config = german (as on some - # servers) 'browser' stems to 'brows' which never matches the - # english-stemmed index. - connection = ActiveRecord::Base.connection - original = connection.select_value('SHOW default_text_search_config') - connection.execute("SET default_text_search_config = 'german'") - assert Package.lexical_overlap?('browser') - ensure - connection&.execute("SET default_text_search_config = '#{original}'") - end - - test 'lexical_overlap? detects real words vs gibberish' do - # words present in fixture package descriptions - assert Package.lexical_overlap?('browser') - assert Package.lexical_overlap?('email client for vim syntax') - assert Package.lexical_overlap?('the browser') # stopwords are ignored - # gibberish, numbers and empty strings share no vocabulary - assert_not Package.lexical_overlap?('asdfgh zzzz') - assert_not Package.lexical_overlap?('123 !!!') - assert_not Package.lexical_overlap?('the') # stopword-only query - assert_not Package.lexical_overlap?('') - assert_not Package.lexical_overlap?(nil) - end - - test 'nearest_to_text embeds the query and searches' do - vector = Array.new(384, 0.25) - embed_called_with = nil - with_stubbed_method(Vectorizer, :embed, - lambda { |text| - embed_called_with = text - vector - }) do - received = [] - results = with_stubbed_method(Package, :nearest_neighbors, - lambda { |_field, received_vector, **_opts| - received << received_vector - [packages(:vim)] - }) do - Package.nearest_to_text('some search text') - end - assert_equal [packages(:vim)], results - assert_equal 'some search text', embed_called_with - assert_equal [vector], received - end - end + # test "the truth" do + # assert true + # end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 1d786a4..4889a19 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) require 'rails/test_help' @@ -17,23 +15,6 @@ module ActiveSupport class TestCase # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. fixtures :all - - # Replace a class-level method with a fixed return value or lambda - # for the duration of the block. Used to keep tests away from - # external services like the vector embedding web service. - def with_stubbed_method(klass, method_name, value_or_lambda) - original = klass.method(method_name) - klass.define_singleton_method(method_name) do |*args, **kwargs| - if value_or_lambda.respond_to?(:call) - value_or_lambda.call(*args, **kwargs) - else - value_or_lambda - end - end - yield - ensure - klass.define_singleton_method(method_name, original) - end end end