diff --git a/Dockerfile b/Dockerfile index e451fa2..177771d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,10 @@ ARG RUBY_VERSION=3.4.7 ARG NODE_VERSION=22.16.0 ARG BUN_VERSION=1.3.9 -ARG GIT_REV=unknown # --- Stage 1: Grab Node.js Binaries --- # We use the official image as a source for pre-compiled binaries -FROM docker.io/library/node:${NODE_VERSION} AS node_source +FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source # --- Stage 2: Base (Runtime Environment) --- FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base @@ -37,13 +36,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update -qq && \ apt-get install --no-install-recommends -y \ - build-essential git libyaml-dev pkg-config python-is-python3 + build-essential git libyaml-dev node-gyp pkg-config python-is-python3 -# Install Node.js and npm from node_source -COPY --from=node_source /usr/local/ /usr/local/ +# "Heist" Node.js and npm from the node_source stage (build-time only) +COPY --from=node_source /usr/local/bin/node /usr/local/bin/node +COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules -# Install Bun -RUN npm install -g bun@${BUN_VERSION} +# Re-link npm and install Bun (build-time only) +RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ + npm install -g bun@${BUN_VERSION} # 1. Install Gems (Separate COPY for maximum caching) COPY Gemfile Gemfile.lock ./ @@ -79,11 +80,8 @@ RUN groupadd --system --gid 1000 rails && \ useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \ chown -R rails:rails db log tmp public -# Redeclare so it is in scope in this final stage -ARG GIT_REV=unknown - -# Create versioninfo with git revision and build timestamp -RUN printf 'git-rev: %s\nbuilt-at: %s\n' "$GIT_REV" "$(date +'%Y-%m-%d %H:%M:%S %Z')" > public/versioninfo +# Create versioninfo with build timestamp +RUN date +"%Y-%m-%d %H:%M:%S %Z" > public/versioninfo USER 1000:1000 diff --git a/Gemfile b/Gemfile index c7a6922..c1e1ea4 100644 --- a/Gemfile +++ b/Gemfile @@ -41,7 +41,8 @@ gem 'mini_magick' # Reduces boot times through caching; required in config/boot.rb gem 'bootsnap', require: false - +# Structured logging for production +gem 'lograge' # Deploy this application anywhere as a Docker container [https://kamal-deploy.org] gem 'kamal', require: false @@ -108,8 +109,6 @@ group :test do # Use system testing [https://guides.rubyonrails.org/testing.html#system-testing] gem 'capybara' gem 'selenium-webdriver' - # Provides the assigns() test helper extracted from modern Rails - gem 'rails-controller-testing' end # TODO… https://github.com/galetahub/simple-captcha @@ -125,10 +124,6 @@ gem 'will_paginate-foundation' # Full-text search in PostgreSQL gem 'pg_search' -# Nearest neighbor search for PostgreSQL using the pgvector extension -# https://github.com/ankane/neighbor -gem 'neighbor' - # Use SLIM as our templating language gem 'slim-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 65bbd8a..202b1e2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -224,6 +224,11 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) logger (1.7.0) + lograge (0.14.0) + actionpack (>= 4) + activesupport (>= 4) + railties (>= 4) + request_store (~> 1.0) loofah (2.25.1) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -250,8 +255,6 @@ GEM minitest (>= 5.0, < 7) ruby-progressbar msgpack (1.8.0) - neighbor (1.2.0) - activerecord (>= 7.2) nenv (0.3.0) net-http (0.9.1) uri (>= 0.11.1) @@ -381,10 +384,6 @@ GEM activesupport (= 8.1.3) bundler (>= 1.15.0) railties (= 8.1.3) - rails-controller-testing (1.0.5) - actionpack (>= 5.0.1.rc1) - actionview (>= 5.0.1.rc1) - activesupport (>= 5.0.1.rc1) rails-dom-testing (2.3.0) activesupport (>= 5.0.0) minitest @@ -428,6 +427,8 @@ GEM regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) + request_store (1.7.0) + rack (>= 1.4) responders (3.2.0) actionpack (>= 7.0) railties (>= 7.0) @@ -584,10 +585,10 @@ DEPENDENCIES jbuilder kamal listen (~> 3.5) + lograge mini_magick minitest-rails minitest-reporters - neighbor omniauth omniauth-rails_csrf_protection omniauth_openid_connect @@ -596,7 +597,6 @@ DEPENDENCIES propshaft puma (>= 5.0) rails (~> 8.1.0) - rails-controller-testing rails-erd rails-healthcheck rails_icons (~> 1.3) diff --git a/Makefile b/Makefile index f2799a9..b83b2e3 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,9 @@ -include .env.production .env.development -REGISTRY := git.workaround.org/chaas +REGISTRY := registry.coolify1.workaround.org -stage: TAG=latest +stage: TAG=stage stage: build tag push curl -s -H "Authorization: Bearer $(COOLIFY_STAGE_DEPLOY_TOKEN)" \ -X POST "$(COOLIFY_STAGE_DEPLOY_WEBHOOK)" @@ -15,11 +15,10 @@ prod: build tag push -X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)" build: - docker build --build-arg GIT_REV=$(shell git rev-parse --short HEAD) -t debshots-$(TAG) . + podman build -t debshots-$(TAG) . tag: - docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG) + podman tag debshots-$(TAG) $(REGISTRY)/debshots-web:$(TAG) push: - docker push $(REGISTRY)/debshots:$(TAG) - + podman push $(REGISTRY)/debshots-web:$(TAG) diff --git a/README.md b/README.md index c330d9c..55facc4 100644 --- a/README.md +++ b/README.md @@ -15,42 +15,6 @@ https://salsa.debian.org/debian/debshots/-/issues Check out the doc/README.Development.md -## Package search - -The package search combines several strategies, tried in this order: - -1. **Exact package name**: Searching for `vim` shows exactly this - package without calling any external service. -2. **Name prefix matches**: Packages whose name starts with the query - rank first, e.g. searching for `sqlite` promotes `sqlite3`, - `sqlitebrowser` and friends above all other results. -3. **Compound word splitting**: Packages whose name contains *all* - query tokens match too, so `sqlite browser` finds `sqlitebrowser`. -4. **Semantic search**: The query is embedded via the vector service - (all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among - the package description embeddings are returned, using the pgvector - extension with an HNSW cosine index. Embeddings are computed from - the package description and long description and can be (re-)built - with `bin/rails debshots:compute_vectors`. -5. **Full-text fallback**: If the vector service cannot be reached, - classic PostgreSQL full-text search takes over. - -Queries that share no vocabulary with the package data (gibberish, -keyboard mash) return an honest empty result instead of random -semantic matches. - -**Why is this package in my results?** Each result knows which search -strategy produced it. The controller exposes this as -`@search_sources` (a hash mapping package ids to `:exact`, -`:name_prefix`, `:name_contains`, `:semantic` or `:fulltext`). For -debugging you can render a small badge on every result card by adding -`show_sources=1` to the URL, e.g. -`https://screenshots.debian.net/packages?search=sqlite&show_sources=1`. - -Package pages also show semantically **related packages**, computed as -a nearest neighbor query against the package's own description -embedding - no external service call involved. - ## Deployment Read the doc/README.Installation.md diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 602595b..b242467 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -30,9 +30,6 @@ class PackagesController < ApplicationController @screenshots = @package.screenshots.accessible_by(current_ability, :view).paginate( page: @page, per_page: 6 ) - # Semantic recommendations based on the package's own description - # embedding - a plain nearest neighbor query, no service call. - @related_packages = @package.related_packages end end @@ -362,15 +359,10 @@ class PackagesController < ApplicationController def query_packages packages = Package # .order(visits: :desc) - # Maps package id to the search strategy that put it into the - # result list: :exact, :name_prefix, :name_contains, :semantic or - # :fulltext. Empty when browsing without a search term. - @search_sources = {} - # text search if params[:search].present? logger.debug "Searching for #{params[:search]}" - packages = search_packages(params[:search]) + packages = packages.general_search(params[:search]) end case params[:show] @@ -386,56 +378,6 @@ class PackagesController < ApplicationController packages end - # Text search for packages: an exact package name match wins without - # touching the vector service. Otherwise packages whose name starts - # with the query (or contains all query tokens, catching compound - # words like "sqlite browser" -> "sqlitebrowser") are promoted above - # the semantic (vector) nearest neighbor results. Classic PostgreSQL - # full-text search kicks in when the vector service cannot be reached. - MAX_PROMOTED_NAME_MATCHES = 8 - MAX_SEMANTIC_RESULTS = 100 - private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS - - def search_packages(query) - exact_match = Package.find_by(name: query) - if exact_match - @search_sources[exact_match.id] = :exact - return Package.where(id: exact_match.id) - end - - prefix_matches = Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES).to_a - token_matches = Package.name_contains_all(query.to_s.split(/\s+/)).limit( - MAX_PROMOTED_NAME_MATCHES - ).to_a - prefix_matches.each { |package| @search_sources[package.id] ||= :name_prefix } - token_matches.each { |package| @search_sources[package.id] ||= :name_contains } - - semantic = [] - if Package.lexical_overlap?(query) - begin - semantic = Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a - semantic.each { |package| @search_sources[package.id] ||= :semantic } - rescue Vectorizer::Error => e - logger.error "Semantic search failed: #{e.message}" - flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.' - semantic = Package.general_search(query).limit(100).to_a - semantic.each { |package| @search_sources[package.id] ||= :fulltext } - end - else - logger.debug "Query '#{query}' has no lexical overlap with package data" - end - - # Deduplicate by NAME, not id: if the database ever contains - # duplicate rows (see the packages_name_key incident) users should - # not see the same package twice. - results = (prefix_matches + token_matches + semantic).uniq(&:name) - - # Queries that share no vocabulary with our package data can produce - # no meaningful results. Skip the vector service round-trip and show - # an honest empty result instead of random packages. - results.empty? ? Package.none : results - end - # Store a random identifier and the client's IP address in the session # for later identification. # def create_user_token diff --git a/app/helpers/packages_helper.rb b/app/helpers/packages_helper.rb index b742a6b..d7dec2c 100644 --- a/app/helpers/packages_helper.rb +++ b/app/helpers/packages_helper.rb @@ -1,22 +1,4 @@ module PackagesHelper - # Search result origin badges are only shown when explicitly - # requested via ?show_sources=1 - def search_sources_visible? - params[:show_sources].present? - end - - # Human readable label for the search strategy that put a package - # into the search results (see PackagesController#search_packages) - def search_source_label(source) - { - exact: 'exact match', - name_prefix: 'name match', - name_contains: 'name match', - semantic: 'similar', - fulltext: 'full-text' - }[source] - end - # Return the description and/or the version of the package def screenshot_caption(screenshot) str = [] diff --git a/app/models/package.rb b/app/models/package.rb index c2e31de..441f5c4 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -2,9 +2,6 @@ require 'open-uri' # allows to load URLs using open() require 'json' require 'deb_importer' -# Search and embedding logic keeps accumulating here; the class is -# cohesive enough that splitting it is not worth the indirection. -# rubocop:disable Metrics/ClassLength class Package < ApplicationRecord # PostgreSQL-based full-text search: # https://github.com/Casecommons/pg_search @@ -29,11 +26,6 @@ class Package < ApplicationRecord inverse_of: :package, dependent: :destroy - # Nearest neighbor search using pgvector. The "embedding" column stores - # a vector computed from description + long_description by an external - # web service (see Vectorizer and lib/tasks/vectorize_packages.rake). - has_neighbors :embedding - # default_scope { # order('name ASC') # } @@ -48,103 +40,6 @@ class Package < ApplicationRecord long_description.split(/\n\.\n/).first if long_description end - # The text that the embedding vector is computed from: description - # and long description combined. - def embedding_text - [description, long_description].compact.join("\n") - end - - # Compute and store the embedding vector for this package by asking - # the external vector service. Raises Vectorizer::Error on failure. - # Returns true if a vector was saved. - def update_embedding! - text = embedding_text - return false if text.blank? - - update_column(:embedding, Vectorizer.embed(text)) - end - - # Return a relation of packages whose stored embeddings are closest to - # the given query vector (an array of floats). Records get a - # "neighbor_distance" attribute added (cosine distance, lower is better). - # The returned relation can still be chained (e.g. filtered or paginated). - # - # Only returns packages that already have an embedding computed. - def self.nearest_to_vector(vector, limit: nil) - neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine') - limit ? neighbors.limit(limit) : neighbors - end - - # Turn a free-text search string into a vector via the external web - # service and find the closest packages. - def self.nearest_to_text(text, limit: nil) - return none if text.blank? - - nearest_to_vector(Vectorizer.embed(text), limit: limit) - end - - # Packages whose name starts with the given string - used to promote - # obvious matches like "sqlite3" for a "sqlite" search. - def self.name_starts_with(text) - return none if text.blank? - - where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name) - end - - # Packages whose name contains all given tokens as substrings. Catches - # compound words split by the user, e.g. "sqlite browser" finding the - # "sqlitebrowser" package. - def self.name_contains_all(tokens) - tokens = tokens.select { |token| token.length >= 2 } - return none if tokens.empty? - - scope = self - tokens.each do |token| - scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\') - end - scope.order(:name) - end - - def self.escape_for_like(text) - # Escape LIKE wildcards so that package name characters like "+" or "." - # cannot break the pattern - text.gsub(/[\\%_]/) { |char| "\\#{char}" } - end - - # Packages semantically similar to this one, based on the stored - # description embedding. Returns an empty relation for packages that - # have no embedding (yet). - def related_packages(limit: 6) - return Package.none unless embedding - - Package.nearest_to_vector(embedding, limit: limit + 1) - .where.not(id: id).limit(limit) - end - - # Check whether the given text shares at least one lexeme with any - # package name/description in the database. Queries without such an - # overlap (gibberish, keyboard mash, unsupported languages) can - # neither match full-text nor produce meaningful semantic results - - # the embedding model maps unknown tokens close to the corpus - # centroid, making them look like good matches. - def self.lexical_overlap?(text) - tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) } - return false if tokens.empty? - - # Pin the same text search dictionary that the packages_fts index - # uses - the server-wide default can be anything. - tsquery = tokens.map do |token| - sanitize_sql(["plainto_tsquery('english', ?)", token]) - end.join(' || ') - # The expression must match the packages_fts GIN index definition - # so that Postgres can use the index. - where( - "setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \ - "setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \ - "setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})" - ).exists? - end - # Return a query of all packages that have screenshots def self.with_screenshots # Query for all packages who's ID appears in a screenshot's "package_id" field @@ -218,4 +113,3 @@ class Package < ApplicationRecord end end end -# rubocop:enable Metrics/ClassLength diff --git a/app/views/packages/browse.slim b/app/views/packages/browse.slim index b8e09c9..84d8599 100644 --- a/app/views/packages/browse.slim +++ b/app/views/packages/browse.slim @@ -14,7 +14,7 @@ - if @packages.any? - if @view_style==:grid .grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium" - - @packages.each do |pkg| + - @packages.all.each do |pkg| .cell.pkgcard data-equalizer-watch=true a.black href=package_path(name: pkg.name) .image @@ -24,9 +24,6 @@ = pkg.name .text = pkg.description - - if search_sources_visible? && @search_sources[pkg.id] - .text.search-source - span.label.secondary = search_source_label(@search_sources[pkg.id]) - elsif @view_style==:list - @packages.to_a.each do |pkg| .grid-x.grid-margin-x.listview @@ -40,9 +37,6 @@ a href=package_path(name: pkg.name) =pkg.name p =pkg.description - - if search_sources_visible? && @search_sources[pkg.id] - p - span.label.secondary = search_source_label(@search_sources[pkg.id]) .listview.longdescription = pkg.long_description_first_paragraph diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index 7443302..03c9609 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -70,19 +70,6 @@ / #metadata is the anchor for this sticky container. = render(partial: 'details_rightbox', locals: {pkg: @package}) - / Semantically similar packages, based on the package's description embedding - - if @related_packages.any? - #related - h3 Related packages - .grid-x.grid-margin-x.medium-up-2.large-up-3 data-equalizer=true - - @related_packages.each do |pkg| - .cell.pkgcard data-equalizer-watch=true - a.black href=package_path(name: pkg.name) - .text.pkgname - = pkg.name - .text.small - = pkg.description - javascript: // Where to send POST requests for uploads let ajax_upload_url = '#{{upload_receive_json_path}}'; diff --git a/config/environments/production.rb b/config/environments/production.rb index b1c0f32..ced9b3c 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -37,6 +37,15 @@ Rails.application.configure do config.log_tags = [:request_id] config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) + # Lograge - structured request logging + config.lograge.enabled = true + config.lograge.custom_options = lambda do |event| + { + request_id: event.payload[:request_id], + user_id: event.payload[:user_id] + } + end + # Change to "debug" to log everything (including potentially personally-identifiable information!). config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info') diff --git a/config/initializers/will_paginate.rb b/config/initializers/will_paginate.rb deleted file mode 100644 index e36b7b3..0000000 --- a/config/initializers/will_paginate.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -# Enable will_paginate's Array#paginate so that merged search results -# (name prefix matches + semantic results) can be paginated like -# ActiveRecord relations. -require 'will_paginate/array' diff --git a/db/migrate/20260821203833_install_neighbor_vector.rb b/db/migrate/20260821203833_install_neighbor_vector.rb deleted file mode 100644 index d591f1c..0000000 --- a/db/migrate/20260821203833_install_neighbor_vector.rb +++ /dev/null @@ -1,5 +0,0 @@ -class InstallNeighborVector < ActiveRecord::Migration[8.1] - def change - enable_extension "vector" - end -end diff --git a/db/migrate/20260821210000_add_embedding_to_packages.rb b/db/migrate/20260821210000_add_embedding_to_packages.rb deleted file mode 100644 index 62825f9..0000000 --- a/db/migrate/20260821210000_add_embedding_to_packages.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -# Adds a vector column to store text embeddings of the package -# description and long description. The vectors are computed by an -# external web service and used for nearest neighbor search. -# -# The pgvector extension is enabled by the InstallNeighborVector migration. -class AddEmbeddingToPackages < ActiveRecord::Migration[8.1] - def change - add_column :packages, :embedding, :vector, limit: 384 - - # HNSW index for fast approximate nearest neighbor searches - add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops - end -end diff --git a/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb b/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb deleted file mode 100644 index 89efc03..0000000 --- a/db/migrate/20260904120000_add_unique_name_constraint_to_packages.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -# Packages must be unique by name - the same package is listed once per -# architecture/component in the Debian archive and the importer looks it up -# by name before inserting, so anything that lets two rows for one name appear -# (e.g. concurrent import runs) breaks the site. Production had accidentally -# relied on a constraint that was only ever added ad-hoc on some databases -# and never tracked as a migration; this makes db:migrate the single source -# of truth. -class AddUniqueNameConstraintToPackages < ActiveRecord::Migration[8.1] - def up - add_unique_constraint :packages, :name, name: :packages_name_key unless constraint_exists? - end - - def down - remove_unique_constraint :packages, :name, name: :packages_name_key if constraint_exists? - end - - private - - def constraint_exists? - select_value(<<~SQL.squish) - SELECT 1 - FROM pg_constraint - WHERE conrelid = 'packages'::regclass AND conname = 'packages_name_key' AND contype = 'u' - SQL - end -end \ No newline at end of file diff --git a/db/schema.rb b/db/schema.rb index 433a8e2..2c57b69 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,131 +10,133 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[8.1].define(version: 2026_09_04_120000) do - # These are extensions that must be enabled in order to support this database - enable_extension "pg_catalog.plpgsql" - enable_extension "vector" +ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do + # These are extensions that must be enabled to support this database + enable_extension 'pg_catalog.plpgsql' - create_table "action_mailbox_inbound_emails", force: :cascade do |t| - t.datetime "created_at", null: false - t.string "message_checksum", null: false - t.string "message_id", null: false - t.integer "status", default: 0, null: false - t.datetime "updated_at", null: false - t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true + create_table 'action_mailbox_inbound_emails', force: :cascade do |t| + t.integer 'status', default: 0, null: false + t.string 'message_id', null: false + t.string 'message_checksum', null: false + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['message_id', 'message_checksum'], +name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true end - create_table "action_text_rich_texts", force: :cascade do |t| - t.text "body" - t.datetime "created_at", null: false - t.string "name", null: false - t.bigint "record_id", null: false - t.string "record_type", null: false - t.datetime "updated_at", null: false - t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true + create_table 'action_text_rich_texts', force: :cascade do |t| + t.string 'name', null: false + t.text 'body' + t.string 'record_type', null: false + t.bigint 'record_id', null: false + t.datetime 'created_at', null: false + t.datetime 'updated_at', null: false + t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness', +unique: true end - create_table "active_storage_attachments", force: :cascade do |t| - t.bigint "blob_id", null: false - t.datetime "created_at", precision: nil, null: false - t.string "name", null: false - t.bigint "record_id", null: false - t.string "record_type", null: false - t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id" - t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true + create_table 'active_storage_attachments', force: :cascade do |t| + t.string 'name', null: false + t.string 'record_type', null: false + t.bigint 'record_id', null: false + t.bigint 'blob_id', null: false + t.datetime 'created_at', precision: nil, null: false + t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id' + t.index ['record_type', 'record_id', 'name', 'blob_id'], +name: 'index_active_storage_attachments_uniqueness', unique: true end - create_table "active_storage_blobs", force: :cascade do |t| - t.bigint "byte_size", null: false - t.string "checksum" - t.string "content_type" - t.datetime "created_at", precision: nil, null: false - t.string "filename", null: false - t.string "key", null: false - t.text "metadata" - t.string "service_name", null: false - t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true + create_table 'active_storage_blobs', force: :cascade do |t| + t.string 'key', null: false + t.string 'filename', null: false + t.string 'content_type' + t.text 'metadata' + t.bigint 'byte_size', null: false + t.string 'checksum' + t.datetime 'created_at', precision: nil, null: false + t.string 'service_name', null: false + t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true end - create_table "active_storage_variant_records", force: :cascade do |t| - t.bigint "blob_id", null: false - t.string "variation_digest", null: false - t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true + create_table 'active_storage_variant_records', force: :cascade do |t| + t.bigint 'blob_id', null: false + t.string 'variation_digest', null: false + t.index ['blob_id', 'variation_digest'], +name: 'index_active_storage_variant_records_uniqueness', unique: true end - create_table "logs", id: :serial, force: :cascade do |t| - t.datetime "created_at", precision: nil, null: false - t.inet "ip_address" - t.string "level" - t.string "message" - t.integer "package_id" - t.integer "screenshot_id" - t.string "section" - t.datetime "updated_at", precision: nil, null: false - t.integer "user_id" + create_table 'logs', id: :serial, force: :cascade do |t| + t.string 'message' + t.string 'level' + t.string 'section' + t.inet 'ip_address' + t.datetime 'created_at', precision: nil, null: false + t.datetime 'updated_at', precision: nil, null: false + t.integer 'user_id' + t.integer 'package_id' + t.integer 'screenshot_id' end - create_table "packages", id: :serial, force: :cascade do |t| - t.datetime "created_at", precision: nil - t.string "description", limit: 80 - t.vector "embedding", limit: 384 - t.string "homepage", limit: 400 - t.text "long_description" - t.string "maintainer", limit: 100 - t.string "maintainer_email", limit: 100 - t.string "name", null: false - t.string "origin", limit: 80 - t.string "section", limit: 50 - t.datetime "updated_at", precision: nil - t.string "version", limit: 200 - t.integer "visits", default: 0 - t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin - t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw - t.unique_constraint ["name"], name: "packages_name_key" + create_table 'packages', id: :serial, force: :cascade do |t| + t.string 'name', null: false + t.string 'description', limit: 80 + t.string 'section', limit: 50 + t.string 'maintainer', limit: 100 + t.string 'maintainer_email', limit: 100 + t.string 'homepage', limit: 400 + t.string 'version', limit: 200 + t.text 'long_description' + t.string 'origin', limit: 80 + t.datetime 'created_at', precision: nil + t.datetime 'updated_at', precision: nil + t.integer 'visits', default: 0 + t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", +name: 'packages_fts', using: :gin + t.unique_constraint ['name'], name: 'packages_name_key' end - create_table "screenshots", id: :serial, force: :cascade do |t| - t.boolean "approved", default: false, null: false - t.datetime "created_at", precision: nil - t.text "description" - t.boolean "hidden", default: false - t.string "image_fingerprint" - t.integer "package_id" - t.text "simage_data" - t.datetime "updated_at", precision: nil - t.string "uploaderhash", limit: 72 - t.inet "uploaderip" - t.integer "user_id", default: 0 - t.string "version", limit: 50 - t.index ["id", "approved"], name: "id_approved" - t.index ["id", "uploaderhash"], name: "id_uploaderhash" + create_table 'screenshots', id: :serial, force: :cascade do |t| + t.integer 'package_id' + t.string 'version', limit: 50 + t.datetime 'created_at', precision: nil + t.string 'uploaderhash', limit: 72 + t.boolean 'approved', default: false, null: false + t.text 'description' + t.datetime 'updated_at', precision: nil + t.string 'image_fingerprint' + t.integer 'user_id', default: 0 + t.text 'simage_data' + t.inet 'uploaderip' + t.boolean 'hidden', default: false + t.index ['id', 'approved'], name: 'id_approved' + t.index ['id', 'uploaderhash'], name: 'id_uploaderhash' end - create_table "users", id: :serial, force: :cascade do |t| - t.boolean "admin_role", default: false - t.integer "approved_screenshots", default: 0 - t.datetime "created_at", precision: nil, null: false - t.datetime "current_sign_in_at", precision: nil - t.inet "current_sign_in_ip" - t.string "email", default: "", null: false - t.string "encrypted_password", default: "", null: false - t.integer "failed_attempts", default: 0, null: false - t.datetime "last_sign_in_at", precision: nil - t.inet "last_sign_in_ip" - t.datetime "locked_at", precision: nil - t.boolean "moderator_role", default: false - t.text "name" - t.string "provider" - t.boolean "pseudo", default: false - t.integer "rejected_screenshots", default: 0 - t.integer "sign_in_count", default: 0, null: false - t.string "uid" - t.string "unlock_token" - t.datetime "updated_at", precision: nil, null: false - t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true + create_table 'users', id: :serial, force: :cascade do |t| + t.text 'name' + t.datetime 'created_at', precision: nil, null: false + t.datetime 'updated_at', precision: nil, null: false + t.string 'email', default: '', null: false + t.string 'encrypted_password', default: '', null: false + t.integer 'sign_in_count', default: 0, null: false + t.datetime 'current_sign_in_at', precision: nil + t.datetime 'last_sign_in_at', precision: nil + t.inet 'current_sign_in_ip' + t.inet 'last_sign_in_ip' + t.integer 'failed_attempts', default: 0, null: false + t.string 'unlock_token' + t.datetime 'locked_at', precision: nil + t.string 'provider' + t.string 'uid' + t.boolean 'admin_role', default: false + t.boolean 'moderator_role', default: false + t.boolean 'pseudo', default: false + t.integer 'approved_screenshots', default: 0 + t.integer 'rejected_screenshots', default: 0 + t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true end - add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id" - add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id" - add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey" + add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id' + add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id' + add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey' end diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 5569c07..8f84233 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -29,47 +29,11 @@ BLACKLIST_SECTION_PATTERN = [ %r{/?oldlibs$}, %r{/?libdevel$}, %r{/?cli-mono$}, - # Libraries never make useful screenshots - and unlike a "lib*" name - # pattern this does not hit applications like libreoffice (editors) - # or librecad (graphics) - %r{/?libs$}, -] - -# List of regular expressions. If the package's short description -# matches any of these then the package will not be imported. -# Transitional/dummy stub packages exist only to ease upgrades to a -# renamed or split package and never get a useful screenshot. Debian -# has no dedicated metadata field for this - the Developer's Reference -# (6.9.7) documents these exact phrases as the convention package -# maintainers use, and as what deborphan --guess-dummy looks for. -# Deliberately not matching a bare "dummy": too many real packages use -# that word for actual functionality (e.g. xserver-xorg-video-dummy, -# fence-agents-dummy, "double dummy solver" bridge card libraries). -BLACKLIST_DESCRIPTION_PATTERN = [ - /transitional/i, - /dummy package/i, - /dependency package/i, ] # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true -# Postgres advisory lock used to serialize update_from_deb_repos runs. -# Without this, a scheduled and a manual run (or two scheduled runs) can -# overlap: both do find_by+insert for the same name, and the second one -# either violates packages_name_key (aborting the whole import) or - on -# instances that never had that constraint - silently creates a duplicate. -IMPORT_ADVISORY_LOCK_KEY = 0x0DE65507 - -# Whether to delete packages that were not found in any configured -# source during a full import run (e.g. packages Debian has dropped -# entirely, or ones from a suite/architecture no longer configured). -# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking -# "orphaned" can also mean a mirror had a transient fetch problem, so -# update_from_deb_repos refuses to act on it unless every configured -# component/architecture was actually fetched successfully this run. -REMOVE_ORPHANED_PACKAGES = false - namespace :debshots do desc "Import/update package database from configured DEB repositories" @@ -79,23 +43,11 @@ namespace :debshots do stats_updated = 0 stats_removed = 0 - # Every package name encountered in any configured source this run, - # used to find orphans afterwards. A Hash is used as a cheap set. - seen_package_names = {} - # Set to true if any component/architecture could not be fetched, - # so we know not to trust seen_package_names for orphan removal. - sources_incomplete = false - repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO #Rails.logger.level = Logger::DEBUG - unless import_lock_acquired? - Rails.logger.warn 'Another import run already in progress (advisory lock). Skipping.' - exit 0 - end - Rails.logger.info "Importing Debian package information" repositories.each do |repository| @@ -122,7 +74,6 @@ namespace :debshots do packages = release.packages(component, architecture) unless packages Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" - sources_incomplete = true next end @@ -131,11 +82,8 @@ namespace :debshots do packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" - seen_package_names[package[:Package]] = true - if package_name_blacklisted?(package[:Package]) or - package_section_blacklisted?(package[:Section]) or - package_description_blacklisted?(package[:Description]) + if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] @@ -149,37 +97,24 @@ namespace :debshots do next end # if blacklisted - begin - db_package = Package.find_by name: package[:Package] - if db_package # package exists in the database - # check if the parsed package data has a newer version - current_version = DebImporter::Version.new(db_package.version) - new_version = DebImporter::Version.new(package[:Version]) + db_package = Package.find_by name: package[:Package] + if db_package # package exists in the database + # check if the parsed package data has a newer version + current_version = DebImporter::Version.new(db_package.version) + new_version = DebImporter::Version.new(package[:Version]) - Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" - if new_version > current_version - # update the package information from new data - update_data(package, db_package) - Rails.logger.info "Updating package information" - stats_updated += 1 - end - else # new package - create it in the database - Rails.logger.debug "No such package in our database. Creating one." - db_package = Package.new - update_data(package, db_package) - stats_added += 1 - end - rescue ActiveRecord::RecordNotUnique - # Lost a create-race: another process inserted this package name - # between our find and our insert (a web upload, or an import on - # an instance without the advisory lock). Treat it as found and - # update instead of aborting the whole run. - Rails.logger.warn "Package '#{package[:Package]}' was created " \ - 'concurrently. Updating it instead.' - if (db_package = Package.find_by(name: package[:Package])) + Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" + if new_version > current_version + # update the package information from new data update_data(package, db_package) + Rails.logger.info "Updating package information" stats_updated += 1 end + else # new package - create it in the database + Rails.logger.debug "No such package in our database. Creating one." + db_package = Package.new + update_data(package, db_package) + stats_added += 1 end # unless db_package @@ -203,9 +138,6 @@ namespace :debshots do Rails.logger.info "--------------------------------------------" end # repositories.each - stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete) - Rails.logger.info "#{stats_removed} packages removed in total" - end # task desc 'Update long description from i18n file' @@ -344,80 +276,3 @@ def package_section_blacklisted?(section) end return false end - -# Check if a package's short description marks it as a transitional or -# dummy stub package (e.g. "transitional package", "transitional dummy -# package for foo"). These never get a useful screenshot. -def package_description_blacklisted?(description) - BLACKLIST_DESCRIPTION_PATTERN.each do |pattern| - if description.to_s =~ pattern - Rails.logger.debug " > Blacklisted by description ('#{description}' matches '#{pattern}')" - return true - end - end - false -end - -# Remove packages that were not encountered in any configured source -# during this import run. Refuses to act if any component/architecture -# failed to fetch (sources_incomplete) or if seen_names came back empty -# (both would otherwise risk wiping out packages that are still very -# much alive in Debian, just missed because of a network hiccup). -# Returns the number of packages actually removed. -def orphan_removal_blocked?(seen_names, sources_incomplete) - if sources_incomplete - Rails.logger.warn ' > Skipping: not all configured components/architectures could be ' \ - 'fetched this run. Re-run when all sources are reachable.' - return true - end - - if seen_names.empty? - Rails.logger.warn ' > Skipping: no packages were seen this run at all - refusing to ' \ - 'treat that as "everything is orphaned".' - return true - end - - false -end - -def find_orphaned_package_ids(seen_names) - Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids| - ids << id unless seen_names.key?(name) - end -end - -def destroy_orphaned_packages(ids) - removed = 0 - Package.where(id: ids).find_each do |package| - Rails.logger.info " > Removing orphaned package '#{package.name}' from database" - package.destroy - removed += 1 - end - removed -end - -def remove_orphaned_packages(seen_names, sources_incomplete) - Rails.logger.info 'Checking for packages no longer found in any configured source' - return 0 if orphan_removal_blocked?(seen_names, sources_incomplete) - - orphan_ids = find_orphaned_package_ids(seen_names) - - unless REMOVE_ORPHANED_PACKAGES - Rails.logger.info " > #{orphan_ids.size} packages are orphaned (dry run - set " \ - 'REMOVE_ORPHANED_PACKAGES to actually remove them)' - return 0 - end - - destroy_orphaned_packages(orphan_ids) -end - -# Try to take the advisory lock guarding update_from_deb_repos. The lock lives -# on this database connection/session, so it is released automatically when the -# process exits - no explicit unlock needed. Returns true if this run is the -# one that may proceed. -def import_lock_acquired? - result = ActiveRecord::Base.connection.execute( - "SELECT pg_try_advisory_lock(#{IMPORT_ADVISORY_LOCK_KEY})" - ) - result.first['pg_try_advisory_lock'] == true -end diff --git a/lib/tasks/vectorize_packages.rake b/lib/tasks/vectorize_packages.rake deleted file mode 100644 index aee5d45..0000000 --- a/lib/tasks/vectorize_packages.rake +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -# Long rake task bodies are normal; the block-length cop is meant for -# application code. -# rubocop:disable Metrics/BlockLength -namespace :debshots do - desc 'Compute embedding vectors for all packages based on their description ' \ - 'and long description (missing ones only unless FORCE=1)' - task compute_vectors: :environment do - logger = Logger.new($stdout) - logger.level = Logger::INFO - - force = ENV['FORCE'].present? - # Packages without any description text cannot be embedded at all - - # exclude them so they do not fail on every single run. - base_scope = Package.where("COALESCE(description, '') <> '' OR " \ - "COALESCE(long_description, '') <> ''") - scope = force ? base_scope : base_scope.where(embedding: nil) - total = scope.count - - if total.zero? - logger.info 'No packages need vector computation.' - next - end - - logger.info "Computing vectors for #{total} packages " \ - "(#{force ? 'recomputing' : 'missing only'}, " \ - "service: #{Vectorizer.service_url})" - - done = 0 - failed = 0 - - # Remember the most recent SQL statement so failures can be traced - # to the exact query that triggered them. - last_sql = nil - ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| - last_sql = args.last[:sql] - end - - scope.find_each do |package| - begin - raise ArgumentError, 'no description text' if package.embedding_text.blank? - - package.update_embedding! - done += 1 - rescue StandardError => e - # A single broken package must never stop the whole run - log - # enough detail to diagnose it and move on. - failed += 1 - logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}" - logger.error "Last SQL: #{last_sql}" - logger.error e.backtrace.first(15).join("\n") if e.backtrace - end - - processed = done + failed - next unless (processed % 100).zero? || processed == total - - percent = (processed.to_f / total * 100).round(1) - logger.info "Progress: #{processed}/#{total} (#{percent}%)" - end - - logger.info "Done. #{done} vectors computed, #{failed} failures." - end -end -# rubocop:enable Metrics/BlockLength diff --git a/lib/vectorizer.rb b/lib/vectorizer.rb deleted file mode 100644 index 84df3c4..0000000 --- a/lib/vectorizer.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'json' - -# Client for the external web service that turns text into embedding -# vectors for nearest neighbor search (pgvector). -# -# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...] -# -# The service URL can be overridden with the VECTOR_SERVICE_URL -# environment variable. -class Vectorizer - # Raised when the vector service cannot be reached or returns garbage - class Error < StandardError; end - - DEFAULT_URL = 'https://vector-search.workaround.org/vector' - - class << self - # Compute the embedding vector for the given text. - # Returns an array of floats. - def embed(text) - new.embed(text) - end - - def service_url - ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL) - end - end - - def embed(text) - response = post_text(text) - - unless response.is_a?(Net::HTTPSuccess) - raise Error, "Vector service returned HTTP #{response.code}" - end - - JSON.parse(response.body).fetch('vector') - rescue JSON::ParserError => e - raise Error, "Vector service returned invalid JSON: #{e.message}" - rescue KeyError - raise Error, 'Vector service response did not contain a "vector" field' - end - - private - - def post_text(text) - uri = URI.parse(self.class.service_url) - request = build_request(uri, text) - - options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 } - Net::HTTP.start(uri.host, uri.port, **options) do |http| - http.request(request) - end - rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e - raise Error, "Could not reach vector service: #{e.message}" - end - - def build_request(uri, text) - request = Net::HTTP::Post.new(uri.request_uri) - request['Content-Type'] = 'application/json' - request.body = { text: text }.to_json - - # Optional Bearer token if the service runs with API_SECRET set - secret = ENV['VECTOR_SERVICE_SECRET'] - request['Authorization'] = "Bearer #{secret}" if secret.present? - - request - end -end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index d72ad95..c9eae31 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require 'test_helper' class PackagesControllerTest < ActionController::TestCase @@ -22,118 +20,6 @@ class PackagesControllerTest < ActionController::TestCase assert_select 'div', 'A program to browse web sites.' end - test 'exact package name search wins without calling the vector service' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text) { flunk 'should not call vector search' }) do - get :grid, params: { search: 'vim', show_sources: '1' } - assert_response :success - assert_select 'div', 'vim' - assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources)) - assert_select '.search-source .label', 'exact match' - end - end - - test 'gibberish search shows no results and never calls the vector service' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text) { flunk 'should not call vector search' }) do - get :grid, params: { search: 'asdfgh zzzz' } - assert_response :success - # the grid renders every package in the database when no search - # filter applies - so an empty result must not fall back to that - assert_select 'div', text: /package-\d+/, count: 0 - assert_empty assigns(:search_sources) - end - end - - test 'name prefix matches rank above semantic results' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do - get :grid, params: { search: 'package' } - assert_response :success - # fixtures create package-0..package-99 - prefix hits must render - assert_select 'div', 'package-0' - # ...and before any semantically found package - assert response.body.index('package-0') < response.body.index('vim') - sources = assigns(:search_sources) - assert_equal :name_prefix, sources[Package.find_by(name: 'package-0').id] - assert_equal :semantic, sources[packages(:vim).id] - end - end - - test 'prefix matches and semantic results are deduplicated' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do - get :grid, params: { search: 'fire' } - assert_response :success - assert_select 'div', text: 'firefox', count: 1 - # the higher tier wins the attribution - assert_equal :name_prefix, assigns(:search_sources)[packages(:firefox).id] - end - end - - test 'compound words in search match packages with joined names' do - # "fire fox" should find "firefox" via name substring matching - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.none }) do - get :grid, params: { search: 'fire fox', show_sources: '1' } - assert_response :success - assert_select 'div', text: 'firefox', count: 1 - assert_equal({ packages(:firefox).id => :name_contains }, - assigns(:search_sources)) - assert_select '.search-source .label', 'name match' - end - end - - test 'search source badges stay hidden without show_sources param' do - with_stubbed_method(Package, :nearest_to_text, - ->(_text, **_opts) { Package.none }) do - get :grid, params: { search: 'fire fox' } - assert_response :success - assert_select '.search-source', count: 0 - end - end - - test 'details shows semantically related packages' do - packages(:vim).update_column(:embedding, Array.new(384, 0.25)) - packages(:firefox).update_column(:embedding, Array.new(384, 0.75)) - - get :details, params: { name: 'vim' } - assert_response :success - assert_select '#related .pkgname', text: 'firefox' - end - - test 'details without embedding renders no related section' do - get :details, params: { name: 'vim' } - assert_response :success - assert_select '#related', count: 0 - end - - test 'search uses semantic (vector) results' do - relation = Package.where(name: packages(:vim).name) - with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do - get :grid, params: { search: 'editor' } - assert_response :success - assert_select 'div', 'vim' - assert_equal({ packages(:vim).id => :semantic }, assigns(:search_sources)) - end - end - - # There is deliberately no fallback for an empty vector search - # result: KNN always returns something as long as packages have - # embeddings. Full-text search is only used when the vector service - # cannot be reached. - - test 'search falls back to full-text search when vector service fails' do - with_stubbed_method(Vectorizer, :embed, - ->(_text) { raise Vectorizer::Error, 'service down' }) do - get :grid, params: { search: 'editor' } - assert_response :success - assert_select 'div', 'vim' - assert flash[:error].present? - assert_equal({ packages(:vim).id => :fulltext }, assigns(:search_sources)) - end - end - # test 'should get thumbnail for a package and a desired version' do # # get '/thumbnail-with-version/package-5/0.1' # get thumbnail_with_version_url('package-5', '0.1') diff --git a/test/models/package_test.rb b/test/models/package_test.rb index 36562a2..124f6c4 100644 --- a/test/models/package_test.rb +++ b/test/models/package_test.rb @@ -1,107 +1,7 @@ -# frozen_string_literal: true - require 'test_helper' class PackageTest < ActiveSupport::TestCase - test 'embedding_text combines description and long description' do - package = packages(:firefox) - assert_equal "A web browser\nA program to browse web sites.", package.embedding_text - end - - test 'embedding_text works with missing long description' do - package = packages(:firefox) - package.long_description = nil - assert_equal 'A web browser', package.embedding_text - end - - test 'update_embedding! stores the vector returned by the vector service' do - vector = Array.new(384) { |i| i / 384.0 } - with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do - package = packages(:vim) - assert package.update_embedding! - # The vector type stores float32 so allow small rounding differences - package.reload.embedding.to_a.each_with_index do |actual, i| - assert_in_delta vector[i], actual, 1e-6 - end - end - end - - test 'update_embedding! does nothing without description text' do - package = packages(:firefox) - package.description = nil - package.long_description = nil - with_stubbed_method(Vectorizer, :embed, - ->(_text) { flunk 'should not call the vector service' }) do - assert_not package.update_embedding! - end - end - - test 'nearest_to_text returns none for blank text' do - assert_empty Package.nearest_to_text('') - assert_empty Package.nearest_to_text(nil) - end - - test 'name_starts_with matches name prefixes case-insensitively' do - assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name - assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name - assert_empty Package.name_starts_with('') - assert_empty Package.name_starts_with(nil) - end - - test 'name_contains_all finds compound words split by spaces' do - assert_includes Package.name_contains_all(%w[fire fox]).map(&:name), - packages(:firefox).name - # all tokens must be present in the name - assert_empty Package.name_contains_all(%w[firefox zzzz]) - assert_empty Package.name_contains_all([]) - # single character tokens would flood results - ignore them - assert_empty Package.name_contains_all(['f']) - end - - test 'lexical_overlap? pins the english dictionary regardless of server config' do - # Regression: with default_text_search_config = german (as on some - # servers) 'browser' stems to 'brows' which never matches the - # english-stemmed index. - connection = ActiveRecord::Base.connection - original = connection.select_value('SHOW default_text_search_config') - connection.execute("SET default_text_search_config = 'german'") - assert Package.lexical_overlap?('browser') - ensure - connection&.execute("SET default_text_search_config = '#{original}'") - end - - test 'lexical_overlap? detects real words vs gibberish' do - # words present in fixture package descriptions - assert Package.lexical_overlap?('browser') - assert Package.lexical_overlap?('email client for vim syntax') - assert Package.lexical_overlap?('the browser') # stopwords are ignored - # gibberish, numbers and empty strings share no vocabulary - assert_not Package.lexical_overlap?('asdfgh zzzz') - assert_not Package.lexical_overlap?('123 !!!') - assert_not Package.lexical_overlap?('the') # stopword-only query - assert_not Package.lexical_overlap?('') - assert_not Package.lexical_overlap?(nil) - end - - test 'nearest_to_text embeds the query and searches' do - vector = Array.new(384, 0.25) - embed_called_with = nil - with_stubbed_method(Vectorizer, :embed, - lambda { |text| - embed_called_with = text - vector - }) do - received = [] - results = with_stubbed_method(Package, :nearest_neighbors, - lambda { |_field, received_vector, **_opts| - received << received_vector - [packages(:vim)] - }) do - Package.nearest_to_text('some search text') - end - assert_equal [packages(:vim)], results - assert_equal 'some search text', embed_called_with - assert_equal [vector], received - end - end + # test "the truth" do + # assert true + # end end diff --git a/test/test_helper.rb b/test/test_helper.rb index 1d786a4..4889a19 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - ENV['RAILS_ENV'] ||= 'test' require File.expand_path('../config/environment', __dir__) require 'rails/test_help' @@ -17,23 +15,6 @@ module ActiveSupport class TestCase # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. fixtures :all - - # Replace a class-level method with a fixed return value or lambda - # for the duration of the block. Used to keep tests away from - # external services like the vector embedding web service. - def with_stubbed_method(klass, method_name, value_or_lambda) - original = klass.method(method_name) - klass.define_singleton_method(method_name) do |*args, **kwargs| - if value_or_lambda.respond_to?(:call) - value_or_lambda.call(*args, **kwargs) - else - value_or_lambda - end - end - yield - ensure - klass.define_singleton_method(method_name, original) - end end end