Compare commits

..

No commits in common. "stage" and "main" have entirely different histories.
stage ... main

22 changed files with 164 additions and 965 deletions

View file

@ -5,11 +5,10 @@
ARG RUBY_VERSION=3.4.7
ARG NODE_VERSION=22.16.0
ARG BUN_VERSION=1.3.9
ARG GIT_REV=unknown
# --- Stage 1: Grab Node.js Binaries ---
# We use the official image as a source for pre-compiled binaries
FROM docker.io/library/node:${NODE_VERSION} AS node_source
FROM docker.io/library/node:${NODE_VERSION}-slim AS node_source
# --- Stage 2: Base (Runtime Environment) ---
FROM docker.io/library/ruby:${RUBY_VERSION}-slim AS base
@ -37,13 +36,15 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
apt-get update -qq && \
apt-get install --no-install-recommends -y \
build-essential git libyaml-dev pkg-config python-is-python3
build-essential git libyaml-dev node-gyp pkg-config python-is-python3
# Install Node.js and npm from node_source
COPY --from=node_source /usr/local/ /usr/local/
# "Heist" Node.js and npm from the node_source stage (build-time only)
COPY --from=node_source /usr/local/bin/node /usr/local/bin/node
COPY --from=node_source /usr/local/lib/node_modules /usr/local/lib/node_modules
# Install Bun
RUN npm install -g bun@${BUN_VERSION}
# Re-link npm and install Bun (build-time only)
RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \
npm install -g bun@${BUN_VERSION}
# 1. Install Gems (Separate COPY for maximum caching)
COPY Gemfile Gemfile.lock ./
@ -79,11 +80,8 @@ RUN groupadd --system --gid 1000 rails && \
useradd rails --uid 1000 --gid 1000 --create-home --shell /bin/bash && \
chown -R rails:rails db log tmp public
# Redeclare so it is in scope in this final stage
ARG GIT_REV=unknown
# Create versioninfo with git revision and build timestamp
RUN printf 'git-rev: %s\nbuilt-at: %s\n' "$GIT_REV" "$(date +'%Y-%m-%d %H:%M:%S %Z')" > public/versioninfo
# Create versioninfo with build timestamp
RUN date +"%Y-%m-%d %H:%M:%S %Z" > public/versioninfo
USER 1000:1000

View file

@ -41,7 +41,8 @@ gem 'mini_magick'
# Reduces boot times through caching; required in config/boot.rb
gem 'bootsnap', require: false
# Structured logging for production
gem 'lograge'
# Deploy this application anywhere as a Docker container [https://kamal-deploy.org]
gem 'kamal', require: false
@ -108,8 +109,6 @@ group :test do
# Use system testing [https://guides.rubyonrails.org/testing.html#system-testing]
gem 'capybara'
gem 'selenium-webdriver'
# Provides the assigns() test helper extracted from modern Rails
gem 'rails-controller-testing'
end
# TODO… https://github.com/galetahub/simple-captcha
@ -125,10 +124,6 @@ gem 'will_paginate-foundation'
# Full-text search in PostgreSQL
gem 'pg_search'
# Nearest neighbor search for PostgreSQL using the pgvector extension
# https://github.com/ankane/neighbor
gem 'neighbor'
# Use SLIM as our templating language
gem 'slim-rails'

View file

@ -224,6 +224,11 @@ GEM
rb-fsevent (~> 0.10, >= 0.10.3)
rb-inotify (~> 0.9, >= 0.9.10)
logger (1.7.0)
lograge (0.14.0)
actionpack (>= 4)
activesupport (>= 4)
railties (>= 4)
request_store (~> 1.0)
loofah (2.25.1)
crass (~> 1.0.2)
nokogiri (>= 1.12.0)
@ -250,8 +255,6 @@ GEM
minitest (>= 5.0, < 7)
ruby-progressbar
msgpack (1.8.0)
neighbor (1.2.0)
activerecord (>= 7.2)
nenv (0.3.0)
net-http (0.9.1)
uri (>= 0.11.1)
@ -381,10 +384,6 @@ GEM
activesupport (= 8.1.3)
bundler (>= 1.15.0)
railties (= 8.1.3)
rails-controller-testing (1.0.5)
actionpack (>= 5.0.1.rc1)
actionview (>= 5.0.1.rc1)
activesupport (>= 5.0.1.rc1)
rails-dom-testing (2.3.0)
activesupport (>= 5.0.0)
minitest
@ -428,6 +427,8 @@ GEM
regexp_parser (2.12.0)
reline (0.6.3)
io-console (~> 0.5)
request_store (1.7.0)
rack (>= 1.4)
responders (3.2.0)
actionpack (>= 7.0)
railties (>= 7.0)
@ -584,10 +585,10 @@ DEPENDENCIES
jbuilder
kamal
listen (~> 3.5)
lograge
mini_magick
minitest-rails
minitest-reporters
neighbor
omniauth
omniauth-rails_csrf_protection
omniauth_openid_connect
@ -596,7 +597,6 @@ DEPENDENCIES
propshaft
puma (>= 5.0)
rails (~> 8.1.0)
rails-controller-testing
rails-erd
rails-healthcheck
rails_icons (~> 1.3)

View file

@ -2,9 +2,9 @@
-include .env.production .env.development
REGISTRY := git.workaround.org/chaas
REGISTRY := registry.coolify1.workaround.org
stage: TAG=latest
stage: TAG=stage
stage: build tag push
curl -s -H "Authorization: Bearer $(COOLIFY_STAGE_DEPLOY_TOKEN)" \
-X POST "$(COOLIFY_STAGE_DEPLOY_WEBHOOK)"
@ -15,11 +15,10 @@ prod: build tag push
-X POST "$(COOLIFY_PROD_DEPLOY_WEBHOOK)"
build:
docker build --build-arg GIT_REV=$(shell git rev-parse --short HEAD) -t debshots-$(TAG) .
podman build -t debshots-$(TAG) .
tag:
docker tag debshots-$(TAG) $(REGISTRY)/debshots:$(TAG)
podman tag debshots-$(TAG) $(REGISTRY)/debshots-web:$(TAG)
push:
docker push $(REGISTRY)/debshots:$(TAG)
podman push $(REGISTRY)/debshots-web:$(TAG)

View file

@ -15,42 +15,6 @@ https://salsa.debian.org/debian/debshots/-/issues
Check out the doc/README.Development.md
## Package search
The package search combines several strategies, tried in this order:
1. **Exact package name**: Searching for `vim` shows exactly this
package without calling any external service.
2. **Name prefix matches**: Packages whose name starts with the query
rank first, e.g. searching for `sqlite` promotes `sqlite3`,
`sqlitebrowser` and friends above all other results.
3. **Compound word splitting**: Packages whose name contains *all*
query tokens match too, so `sqlite browser` finds `sqlitebrowser`.
4. **Semantic search**: The query is embedded via the vector service
(all-MiniLM-L6-v2, 384 dimensions) and the nearest neighbors among
the package description embeddings are returned, using the pgvector
extension with an HNSW cosine index. Embeddings are computed from
the package description and long description and can be (re-)built
with `bin/rails debshots:compute_vectors`.
5. **Full-text fallback**: If the vector service cannot be reached,
classic PostgreSQL full-text search takes over.
Queries that share no vocabulary with the package data (gibberish,
keyboard mash) return an honest empty result instead of random
semantic matches.
**Why is this package in my results?** Each result knows which search
strategy produced it. The controller exposes this as
`@search_sources` (a hash mapping package ids to `:exact`,
`:name_prefix`, `:name_contains`, `:semantic` or `:fulltext`). For
debugging you can render a small badge on every result card by adding
`show_sources=1` to the URL, e.g.
`https://screenshots.debian.net/packages?search=sqlite&show_sources=1`.
Package pages also show semantically **related packages**, computed as
a nearest neighbor query against the package's own description
embedding - no external service call involved.
## Deployment
Read the doc/README.Installation.md

View file

@ -30,9 +30,6 @@ class PackagesController < ApplicationController
@screenshots = @package.screenshots.accessible_by(current_ability, :view).paginate(
page: @page, per_page: 6
)
# Semantic recommendations based on the package's own description
# embedding - a plain nearest neighbor query, no service call.
@related_packages = @package.related_packages
end
end
@ -362,15 +359,10 @@ class PackagesController < ApplicationController
def query_packages
packages = Package # .order(visits: :desc)
# Maps package id to the search strategy that put it into the
# result list: :exact, :name_prefix, :name_contains, :semantic or
# :fulltext. Empty when browsing without a search term.
@search_sources = {}
# text search
if params[:search].present?
logger.debug "Searching for #{params[:search]}"
packages = search_packages(params[:search])
packages = packages.general_search(params[:search])
end
case params[:show]
@ -386,56 +378,6 @@ class PackagesController < ApplicationController
packages
end
# Text search for packages: an exact package name match wins without
# touching the vector service. Otherwise packages whose name starts
# with the query (or contains all query tokens, catching compound
# words like "sqlite browser" -> "sqlitebrowser") are promoted above
# the semantic (vector) nearest neighbor results. Classic PostgreSQL
# full-text search kicks in when the vector service cannot be reached.
MAX_PROMOTED_NAME_MATCHES = 8
MAX_SEMANTIC_RESULTS = 100
private_constant :MAX_PROMOTED_NAME_MATCHES, :MAX_SEMANTIC_RESULTS
def search_packages(query)
exact_match = Package.find_by(name: query)
if exact_match
@search_sources[exact_match.id] = :exact
return Package.where(id: exact_match.id)
end
prefix_matches = Package.name_starts_with(query).limit(MAX_PROMOTED_NAME_MATCHES).to_a
token_matches = Package.name_contains_all(query.to_s.split(/\s+/)).limit(
MAX_PROMOTED_NAME_MATCHES
).to_a
prefix_matches.each { |package| @search_sources[package.id] ||= :name_prefix }
token_matches.each { |package| @search_sources[package.id] ||= :name_contains }
semantic = []
if Package.lexical_overlap?(query)
begin
semantic = Package.nearest_to_text(query, limit: MAX_SEMANTIC_RESULTS).to_a
semantic.each { |package| @search_sources[package.id] ||= :semantic }
rescue Vectorizer::Error => e
logger.error "Semantic search failed: #{e.message}"
flash.now['error'] = 'Semantic search is unavailable right now. Showing full-text matches.'
semantic = Package.general_search(query).limit(100).to_a
semantic.each { |package| @search_sources[package.id] ||= :fulltext }
end
else
logger.debug "Query '#{query}' has no lexical overlap with package data"
end
# Deduplicate by NAME, not id: if the database ever contains
# duplicate rows (see the packages_name_key incident) users should
# not see the same package twice.
results = (prefix_matches + token_matches + semantic).uniq(&:name)
# Queries that share no vocabulary with our package data can produce
# no meaningful results. Skip the vector service round-trip and show
# an honest empty result instead of random packages.
results.empty? ? Package.none : results
end
# Store a random identifier and the client's IP address in the session
# for later identification.
# def create_user_token

View file

@ -1,22 +1,4 @@
module PackagesHelper
# Search result origin badges are only shown when explicitly
# requested via ?show_sources=1
def search_sources_visible?
params[:show_sources].present?
end
# Human readable label for the search strategy that put a package
# into the search results (see PackagesController#search_packages)
def search_source_label(source)
{
exact: 'exact match',
name_prefix: 'name match',
name_contains: 'name match',
semantic: 'similar',
fulltext: 'full-text'
}[source]
end
# Return the description and/or the version of the package
def screenshot_caption(screenshot)
str = []

View file

@ -2,9 +2,6 @@ require 'open-uri' # allows to load URLs using open()
require 'json'
require 'deb_importer'
# Search and embedding logic keeps accumulating here; the class is
# cohesive enough that splitting it is not worth the indirection.
# rubocop:disable Metrics/ClassLength
class Package < ApplicationRecord
# PostgreSQL-based full-text search:
# https://github.com/Casecommons/pg_search
@ -29,11 +26,6 @@ class Package < ApplicationRecord
inverse_of: :package,
dependent: :destroy
# Nearest neighbor search using pgvector. The "embedding" column stores
# a vector computed from description + long_description by an external
# web service (see Vectorizer and lib/tasks/vectorize_packages.rake).
has_neighbors :embedding
# default_scope {
# order('name ASC')
# }
@ -48,103 +40,6 @@ class Package < ApplicationRecord
long_description.split(/\n\.\n/).first if long_description
end
# The text that the embedding vector is computed from: description
# and long description combined.
def embedding_text
[description, long_description].compact.join("\n")
end
# Compute and store the embedding vector for this package by asking
# the external vector service. Raises Vectorizer::Error on failure.
# Returns true if a vector was saved.
def update_embedding!
text = embedding_text
return false if text.blank?
update_column(:embedding, Vectorizer.embed(text))
end
# Return a relation of packages whose stored embeddings are closest to
# the given query vector (an array of floats). Records get a
# "neighbor_distance" attribute added (cosine distance, lower is better).
# The returned relation can still be chained (e.g. filtered or paginated).
#
# Only returns packages that already have an embedding computed.
def self.nearest_to_vector(vector, limit: nil)
neighbors = nearest_neighbors(:embedding, vector, distance: 'cosine')
limit ? neighbors.limit(limit) : neighbors
end
# Turn a free-text search string into a vector via the external web
# service and find the closest packages.
def self.nearest_to_text(text, limit: nil)
return none if text.blank?
nearest_to_vector(Vectorizer.embed(text), limit: limit)
end
# Packages whose name starts with the given string - used to promote
# obvious matches like "sqlite3" for a "sqlite" search.
def self.name_starts_with(text)
return none if text.blank?
where('name ILIKE ? ESCAPE ?', "#{escape_for_like(text)}%", '\\').order(:name)
end
# Packages whose name contains all given tokens as substrings. Catches
# compound words split by the user, e.g. "sqlite browser" finding the
# "sqlitebrowser" package.
def self.name_contains_all(tokens)
tokens = tokens.select { |token| token.length >= 2 }
return none if tokens.empty?
scope = self
tokens.each do |token|
scope = scope.where('name ILIKE ? ESCAPE ?', "%#{escape_for_like(token)}%", '\\')
end
scope.order(:name)
end
def self.escape_for_like(text)
# Escape LIKE wildcards so that package name characters like "+" or "."
# cannot break the pattern
text.gsub(/[\\%_]/) { |char| "\\#{char}" }
end
# Packages semantically similar to this one, based on the stored
# description embedding. Returns an empty relation for packages that
# have no embedding (yet).
def related_packages(limit: 6)
return Package.none unless embedding
Package.nearest_to_vector(embedding, limit: limit + 1)
.where.not(id: id).limit(limit)
end
# Check whether the given text shares at least one lexeme with any
# package name/description in the database. Queries without such an
# overlap (gibberish, keyboard mash, unsupported languages) can
# neither match full-text nor produce meaningful semantic results -
# the embedding model maps unknown tokens close to the corpus
# centroid, making them look like good matches.
def self.lexical_overlap?(text)
tokens = text.to_s.split(/\s+/).select { |t| t.match?(/[[:alpha:]]{2,}/) }
return false if tokens.empty?
# Pin the same text search dictionary that the packages_fts index
# uses - the server-wide default can be anything.
tsquery = tokens.map do |token|
sanitize_sql(["plainto_tsquery('english', ?)", token])
end.join(' || ')
# The expression must match the packages_fts GIN index definition
# so that Postgres can use the index.
where(
"setweight(to_tsvector('english', coalesce(name::text, '')), 'A') || " \
"setweight(to_tsvector('english', coalesce(description::text, '')), 'B') || " \
"setweight(to_tsvector('english', coalesce(long_description, '')), 'C') @@ (#{tsquery})"
).exists?
end
# Return a query of all packages that have screenshots
def self.with_screenshots
# Query for all packages who's ID appears in a screenshot's "package_id" field
@ -218,4 +113,3 @@ class Package < ApplicationRecord
end
end
end
# rubocop:enable Metrics/ClassLength

View file

@ -14,7 +14,7 @@
- if @packages.any?
- if @view_style==:grid
.grid-x.grid-margin-x.small-up-1.medium-up-2.large-up-3 data-equalizer=true data-equalize-on="medium"
- @packages.each do |pkg|
- @packages.all.each do |pkg|
.cell.pkgcard data-equalizer-watch=true
a.black href=package_path(name: pkg.name)
.image
@ -24,9 +24,6 @@
= pkg.name
.text
= pkg.description
- if search_sources_visible? && @search_sources[pkg.id]
.text.search-source
span.label.secondary = search_source_label(@search_sources[pkg.id])
- elsif @view_style==:list
- @packages.to_a.each do |pkg|
.grid-x.grid-margin-x.listview
@ -40,9 +37,6 @@
a href=package_path(name: pkg.name)
=pkg.name
p =pkg.description
- if search_sources_visible? && @search_sources[pkg.id]
p
span.label.secondary = search_source_label(@search_sources[pkg.id])
.listview.longdescription
= pkg.long_description_first_paragraph

View file

@ -70,19 +70,6 @@
/ #metadata is the anchor for this sticky container.
= render(partial: 'details_rightbox', locals: {pkg: @package})
/ Semantically similar packages, based on the package's description embedding
- if @related_packages.any?
#related
h3 Related packages
.grid-x.grid-margin-x.medium-up-2.large-up-3 data-equalizer=true
- @related_packages.each do |pkg|
.cell.pkgcard data-equalizer-watch=true
a.black href=package_path(name: pkg.name)
.text.pkgname
= pkg.name
.text.small
= pkg.description
javascript:
// Where to send POST requests for uploads
let ajax_upload_url = '#{{upload_receive_json_path}}';

View file

@ -37,6 +37,15 @@ Rails.application.configure do
config.log_tags = [:request_id]
config.logger = ActiveSupport::TaggedLogging.logger(STDOUT)
# Lograge - structured request logging
config.lograge.enabled = true
config.lograge.custom_options = lambda do |event|
{
request_id: event.payload[:request_id],
user_id: event.payload[:user_id]
}
end
# Change to "debug" to log everything (including potentially personally-identifiable information!).
config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info')

View file

@ -1,6 +0,0 @@
# frozen_string_literal: true
# Enable will_paginate's Array#paginate so that merged search results
# (name prefix matches + semantic results) can be paginated like
# ActiveRecord relations.
require 'will_paginate/array'

View file

@ -1,5 +0,0 @@
class InstallNeighborVector < ActiveRecord::Migration[8.1]
def change
enable_extension "vector"
end
end

View file

@ -1,15 +0,0 @@
# frozen_string_literal: true
# Adds a vector column to store text embeddings of the package
# description and long description. The vectors are computed by an
# external web service and used for nearest neighbor search.
#
# The pgvector extension is enabled by the InstallNeighborVector migration.
class AddEmbeddingToPackages < ActiveRecord::Migration[8.1]
def change
add_column :packages, :embedding, :vector, limit: 384
# HNSW index for fast approximate nearest neighbor searches
add_index :packages, :embedding, using: :hnsw, opclass: :vector_cosine_ops
end
end

View file

@ -1,28 +0,0 @@
# frozen_string_literal: true
# Packages must be unique by name - the same package is listed once per
# architecture/component in the Debian archive and the importer looks it up
# by name before inserting, so anything that lets two rows for one name appear
# (e.g. concurrent import runs) breaks the site. Production had accidentally
# relied on a constraint that was only ever added ad-hoc on some databases
# and never tracked as a migration; this makes db:migrate the single source
# of truth.
class AddUniqueNameConstraintToPackages < ActiveRecord::Migration[8.1]
def up
add_unique_constraint :packages, :name, name: :packages_name_key unless constraint_exists?
end
def down
remove_unique_constraint :packages, :name, name: :packages_name_key if constraint_exists?
end
private
def constraint_exists?
select_value(<<~SQL.squish)
SELECT 1
FROM pg_constraint
WHERE conrelid = 'packages'::regclass AND conname = 'packages_name_key' AND contype = 'u'
SQL
end
end

218
db/schema.rb generated
View file

@ -10,131 +10,133 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[8.1].define(version: 2026_09_04_120000) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
enable_extension "vector"
ActiveRecord::Schema[8.0].define(version: 2023_11_26_173151) do
# These are extensions that must be enabled to support this database
enable_extension 'pg_catalog.plpgsql'
create_table "action_mailbox_inbound_emails", force: :cascade do |t|
t.datetime "created_at", null: false
t.string "message_checksum", null: false
t.string "message_id", null: false
t.integer "status", default: 0, null: false
t.datetime "updated_at", null: false
t.index ["message_id", "message_checksum"], name: "index_action_mailbox_inbound_emails_uniqueness", unique: true
create_table 'action_mailbox_inbound_emails', force: :cascade do |t|
t.integer 'status', default: 0, null: false
t.string 'message_id', null: false
t.string 'message_checksum', null: false
t.datetime 'created_at', null: false
t.datetime 'updated_at', null: false
t.index ['message_id', 'message_checksum'],
name: 'index_action_mailbox_inbound_emails_uniqueness', unique: true
end
create_table "action_text_rich_texts", force: :cascade do |t|
t.text "body"
t.datetime "created_at", null: false
t.string "name", null: false
t.bigint "record_id", null: false
t.string "record_type", null: false
t.datetime "updated_at", null: false
t.index ["record_type", "record_id", "name"], name: "index_action_text_rich_texts_uniqueness", unique: true
create_table 'action_text_rich_texts', force: :cascade do |t|
t.string 'name', null: false
t.text 'body'
t.string 'record_type', null: false
t.bigint 'record_id', null: false
t.datetime 'created_at', null: false
t.datetime 'updated_at', null: false
t.index ['record_type', 'record_id', 'name'], name: 'index_action_text_rich_texts_uniqueness',
unique: true
end
create_table "active_storage_attachments", force: :cascade do |t|
t.bigint "blob_id", null: false
t.datetime "created_at", precision: nil, null: false
t.string "name", null: false
t.bigint "record_id", null: false
t.string "record_type", null: false
t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id"
t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true
create_table 'active_storage_attachments', force: :cascade do |t|
t.string 'name', null: false
t.string 'record_type', null: false
t.bigint 'record_id', null: false
t.bigint 'blob_id', null: false
t.datetime 'created_at', precision: nil, null: false
t.index ['blob_id'], name: 'index_active_storage_attachments_on_blob_id'
t.index ['record_type', 'record_id', 'name', 'blob_id'],
name: 'index_active_storage_attachments_uniqueness', unique: true
end
create_table "active_storage_blobs", force: :cascade do |t|
t.bigint "byte_size", null: false
t.string "checksum"
t.string "content_type"
t.datetime "created_at", precision: nil, null: false
t.string "filename", null: false
t.string "key", null: false
t.text "metadata"
t.string "service_name", null: false
t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true
create_table 'active_storage_blobs', force: :cascade do |t|
t.string 'key', null: false
t.string 'filename', null: false
t.string 'content_type'
t.text 'metadata'
t.bigint 'byte_size', null: false
t.string 'checksum'
t.datetime 'created_at', precision: nil, null: false
t.string 'service_name', null: false
t.index ['key'], name: 'index_active_storage_blobs_on_key', unique: true
end
create_table "active_storage_variant_records", force: :cascade do |t|
t.bigint "blob_id", null: false
t.string "variation_digest", null: false
t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true
create_table 'active_storage_variant_records', force: :cascade do |t|
t.bigint 'blob_id', null: false
t.string 'variation_digest', null: false
t.index ['blob_id', 'variation_digest'],
name: 'index_active_storage_variant_records_uniqueness', unique: true
end
create_table "logs", id: :serial, force: :cascade do |t|
t.datetime "created_at", precision: nil, null: false
t.inet "ip_address"
t.string "level"
t.string "message"
t.integer "package_id"
t.integer "screenshot_id"
t.string "section"
t.datetime "updated_at", precision: nil, null: false
t.integer "user_id"
create_table 'logs', id: :serial, force: :cascade do |t|
t.string 'message'
t.string 'level'
t.string 'section'
t.inet 'ip_address'
t.datetime 'created_at', precision: nil, null: false
t.datetime 'updated_at', precision: nil, null: false
t.integer 'user_id'
t.integer 'package_id'
t.integer 'screenshot_id'
end
create_table "packages", id: :serial, force: :cascade do |t|
t.datetime "created_at", precision: nil
t.string "description", limit: 80
t.vector "embedding", limit: 384
t.string "homepage", limit: 400
t.text "long_description"
t.string "maintainer", limit: 100
t.string "maintainer_email", limit: 100
t.string "name", null: false
t.string "origin", limit: 80
t.string "section", limit: 50
t.datetime "updated_at", precision: nil
t.string "version", limit: 200
t.integer "visits", default: 0
t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))", name: "packages_fts", using: :gin
t.index ["embedding"], name: "index_packages_on_embedding", opclass: :vector_cosine_ops, using: :hnsw
t.unique_constraint ["name"], name: "packages_name_key"
create_table 'packages', id: :serial, force: :cascade do |t|
t.string 'name', null: false
t.string 'description', limit: 80
t.string 'section', limit: 50
t.string 'maintainer', limit: 100
t.string 'maintainer_email', limit: 100
t.string 'homepage', limit: 400
t.string 'version', limit: 200
t.text 'long_description'
t.string 'origin', limit: 80
t.datetime 'created_at', precision: nil
t.datetime 'updated_at', precision: nil
t.integer 'visits', default: 0
t.index "(((setweight(to_tsvector('english'::regconfig, COALESCE((name)::text, ''::text)), 'A'::\"char\") || setweight(to_tsvector('english'::regconfig, COALESCE((description)::text, ''::text)), 'B'::\"char\")) || setweight(to_tsvector('english'::regconfig, COALESCE(long_description, ''::text)), 'C'::\"char\")))",
name: 'packages_fts', using: :gin
t.unique_constraint ['name'], name: 'packages_name_key'
end
create_table "screenshots", id: :serial, force: :cascade do |t|
t.boolean "approved", default: false, null: false
t.datetime "created_at", precision: nil
t.text "description"
t.boolean "hidden", default: false
t.string "image_fingerprint"
t.integer "package_id"
t.text "simage_data"
t.datetime "updated_at", precision: nil
t.string "uploaderhash", limit: 72
t.inet "uploaderip"
t.integer "user_id", default: 0
t.string "version", limit: 50
t.index ["id", "approved"], name: "id_approved"
t.index ["id", "uploaderhash"], name: "id_uploaderhash"
create_table 'screenshots', id: :serial, force: :cascade do |t|
t.integer 'package_id'
t.string 'version', limit: 50
t.datetime 'created_at', precision: nil
t.string 'uploaderhash', limit: 72
t.boolean 'approved', default: false, null: false
t.text 'description'
t.datetime 'updated_at', precision: nil
t.string 'image_fingerprint'
t.integer 'user_id', default: 0
t.text 'simage_data'
t.inet 'uploaderip'
t.boolean 'hidden', default: false
t.index ['id', 'approved'], name: 'id_approved'
t.index ['id', 'uploaderhash'], name: 'id_uploaderhash'
end
create_table "users", id: :serial, force: :cascade do |t|
t.boolean "admin_role", default: false
t.integer "approved_screenshots", default: 0
t.datetime "created_at", precision: nil, null: false
t.datetime "current_sign_in_at", precision: nil
t.inet "current_sign_in_ip"
t.string "email", default: "", null: false
t.string "encrypted_password", default: "", null: false
t.integer "failed_attempts", default: 0, null: false
t.datetime "last_sign_in_at", precision: nil
t.inet "last_sign_in_ip"
t.datetime "locked_at", precision: nil
t.boolean "moderator_role", default: false
t.text "name"
t.string "provider"
t.boolean "pseudo", default: false
t.integer "rejected_screenshots", default: 0
t.integer "sign_in_count", default: 0, null: false
t.string "uid"
t.string "unlock_token"
t.datetime "updated_at", precision: nil, null: false
t.index ["email", "provider"], name: "index_users_on_email_and_provider", unique: true
create_table 'users', id: :serial, force: :cascade do |t|
t.text 'name'
t.datetime 'created_at', precision: nil, null: false
t.datetime 'updated_at', precision: nil, null: false
t.string 'email', default: '', null: false
t.string 'encrypted_password', default: '', null: false
t.integer 'sign_in_count', default: 0, null: false
t.datetime 'current_sign_in_at', precision: nil
t.datetime 'last_sign_in_at', precision: nil
t.inet 'current_sign_in_ip'
t.inet 'last_sign_in_ip'
t.integer 'failed_attempts', default: 0, null: false
t.string 'unlock_token'
t.datetime 'locked_at', precision: nil
t.string 'provider'
t.string 'uid'
t.boolean 'admin_role', default: false
t.boolean 'moderator_role', default: false
t.boolean 'pseudo', default: false
t.integer 'approved_screenshots', default: 0
t.integer 'rejected_screenshots', default: 0
t.index ['email', 'provider'], name: 'index_users_on_email_and_provider', unique: true
end
add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
add_foreign_key "screenshots", "packages", name: "screenshots_package_id_fkey"
add_foreign_key 'active_storage_attachments', 'active_storage_blobs', column: 'blob_id'
add_foreign_key 'active_storage_variant_records', 'active_storage_blobs', column: 'blob_id'
add_foreign_key 'screenshots', 'packages', name: 'screenshots_package_id_fkey'
end

View file

@ -29,47 +29,11 @@ BLACKLIST_SECTION_PATTERN = [
%r{/?oldlibs$},
%r{/?libdevel$},
%r{/?cli-mono$},
# Libraries never make useful screenshots - and unlike a "lib*" name
# pattern this does not hit applications like libreoffice (editors)
# or librecad (graphics)
%r{/?libs$},
]
# List of regular expressions. If the package's short description
# matches any of these then the package will not be imported.
# Transitional/dummy stub packages exist only to ease upgrades to a
# renamed or split package and never get a useful screenshot. Debian
# has no dedicated metadata field for this - the Developer's Reference
# (6.9.7) documents these exact phrases as the convention package
# maintainers use, and as what deborphan --guess-dummy looks for.
# Deliberately not matching a bare "dummy": too many real packages use
# that word for actual functionality (e.g. xserver-xorg-video-dummy,
# fence-agents-dummy, "double dummy solver" bridge card libraries).
BLACKLIST_DESCRIPTION_PATTERN = [
/transitional/i,
/dummy package/i,
/dependency package/i,
]
# Whether to delete a blacklisted package from the database
REMOVE_BLACKLISTED_PACKAGE = true
# Postgres advisory lock used to serialize update_from_deb_repos runs.
# Without this, a scheduled and a manual run (or two scheduled runs) can
# overlap: both do find_by+insert for the same name, and the second one
# either violates packages_name_key (aborting the whole import) or - on
# instances that never had that constraint - silently creates a duplicate.
IMPORT_ADVISORY_LOCK_KEY = 0x0DE65507
# Whether to delete packages that were not found in any configured
# source during a full import run (e.g. packages Debian has dropped
# entirely, or ones from a suite/architecture no longer configured).
# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking
# "orphaned" can also mean a mirror had a transient fetch problem, so
# update_from_deb_repos refuses to act on it unless every configured
# component/architecture was actually fetched successfully this run.
REMOVE_ORPHANED_PACKAGES = false
namespace :debshots do
desc "Import/update package database from configured DEB repositories"
@ -79,23 +43,11 @@ namespace :debshots do
stats_updated = 0
stats_removed = 0
# Every package name encountered in any configured source this run,
# used to find orphans afterwards. A Hash is used as a cheap set.
seen_package_names = {}
# Set to true if any component/architecture could not be fetched,
# so we know not to trust seen_package_names for orphan removal.
sources_incomplete = false
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
#Rails.logger.level = Logger::DEBUG
unless import_lock_acquired?
Rails.logger.warn 'Another import run already in progress (advisory lock). Skipping.'
exit 0
end
Rails.logger.info "Importing Debian package information"
repositories.each do |repository|
@ -122,7 +74,6 @@ namespace :debshots do
packages = release.packages(component, architecture)
unless packages
Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
sources_incomplete = true
next
end
@ -131,11 +82,8 @@ namespace :debshots do
packages.each do |package|
Rails.logger.info "> Package: #{package[:Package]}"
#Rails.logger.debug "Fetching package informaton from the database"
seen_package_names[package[:Package]] = true
if package_name_blacklisted?(package[:Package]) or
package_section_blacklisted?(package[:Section]) or
package_description_blacklisted?(package[:Description])
if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section]
# Should the package get removed from the database?
if REMOVE_BLACKLISTED_PACKAGE
db_package = Package.find_by name: package[:Package]
@ -149,37 +97,24 @@ namespace :debshots do
next
end # if blacklisted
begin
db_package = Package.find_by name: package[:Package]
if db_package # package exists in the database
# check if the parsed package data has a newer version
current_version = DebImporter::Version.new(db_package.version)
new_version = DebImporter::Version.new(package[:Version])
db_package = Package.find_by name: package[:Package]
if db_package # package exists in the database
# check if the parsed package data has a newer version
current_version = DebImporter::Version.new(db_package.version)
new_version = DebImporter::Version.new(package[:Version])
Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
if new_version > current_version
# update the package information from new data
update_data(package, db_package)
Rails.logger.info "Updating package information"
stats_updated += 1
end
else # new package - create it in the database
Rails.logger.debug "No such package in our database. Creating one."
db_package = Package.new
update_data(package, db_package)
stats_added += 1
end
rescue ActiveRecord::RecordNotUnique
# Lost a create-race: another process inserted this package name
# between our find and our insert (a web upload, or an import on
# an instance without the advisory lock). Treat it as found and
# update instead of aborting the whole run.
Rails.logger.warn "Package '#{package[:Package]}' was created " \
'concurrently. Updating it instead.'
if (db_package = Package.find_by(name: package[:Package]))
Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
if new_version > current_version
# update the package information from new data
update_data(package, db_package)
Rails.logger.info "Updating package information"
stats_updated += 1
end
else # new package - create it in the database
Rails.logger.debug "No such package in our database. Creating one."
db_package = Package.new
update_data(package, db_package)
stats_added += 1
end
# unless db_package
@ -203,9 +138,6 @@ namespace :debshots do
Rails.logger.info "--------------------------------------------"
end # repositories.each
stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete)
Rails.logger.info "#{stats_removed} packages removed in total"
end # task
desc 'Update long description from i18n file'
@ -344,80 +276,3 @@ def package_section_blacklisted?(section)
end
return false
end
# Check if a package's short description marks it as a transitional or
# dummy stub package (e.g. "transitional package", "transitional dummy
# package for foo"). These never get a useful screenshot.
def package_description_blacklisted?(description)
BLACKLIST_DESCRIPTION_PATTERN.each do |pattern|
if description.to_s =~ pattern
Rails.logger.debug " > Blacklisted by description ('#{description}' matches '#{pattern}')"
return true
end
end
false
end
# Remove packages that were not encountered in any configured source
# during this import run. Refuses to act if any component/architecture
# failed to fetch (sources_incomplete) or if seen_names came back empty
# (both would otherwise risk wiping out packages that are still very
# much alive in Debian, just missed because of a network hiccup).
# Returns the number of packages actually removed.
def orphan_removal_blocked?(seen_names, sources_incomplete)
if sources_incomplete
Rails.logger.warn ' > Skipping: not all configured components/architectures could be ' \
'fetched this run. Re-run when all sources are reachable.'
return true
end
if seen_names.empty?
Rails.logger.warn ' > Skipping: no packages were seen this run at all - refusing to ' \
'treat that as "everything is orphaned".'
return true
end
false
end
def find_orphaned_package_ids(seen_names)
Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids|
ids << id unless seen_names.key?(name)
end
end
def destroy_orphaned_packages(ids)
removed = 0
Package.where(id: ids).find_each do |package|
Rails.logger.info " > Removing orphaned package '#{package.name}' from database"
package.destroy
removed += 1
end
removed
end
def remove_orphaned_packages(seen_names, sources_incomplete)
Rails.logger.info 'Checking for packages no longer found in any configured source'
return 0 if orphan_removal_blocked?(seen_names, sources_incomplete)
orphan_ids = find_orphaned_package_ids(seen_names)
unless REMOVE_ORPHANED_PACKAGES
Rails.logger.info " > #{orphan_ids.size} packages are orphaned (dry run - set " \
'REMOVE_ORPHANED_PACKAGES to actually remove them)'
return 0
end
destroy_orphaned_packages(orphan_ids)
end
# Try to take the advisory lock guarding update_from_deb_repos. The lock lives
# on this database connection/session, so it is released automatically when the
# process exits - no explicit unlock needed. Returns true if this run is the
# one that may proceed.
def import_lock_acquired?
result = ActiveRecord::Base.connection.execute(
"SELECT pg_try_advisory_lock(#{IMPORT_ADVISORY_LOCK_KEY})"
)
result.first['pg_try_advisory_lock'] == true
end

View file

@ -1,65 +0,0 @@
# frozen_string_literal: true
# Long rake task bodies are normal; the block-length cop is meant for
# application code.
# rubocop:disable Metrics/BlockLength
namespace :debshots do
desc 'Compute embedding vectors for all packages based on their description ' \
'and long description (missing ones only unless FORCE=1)'
task compute_vectors: :environment do
logger = Logger.new($stdout)
logger.level = Logger::INFO
force = ENV['FORCE'].present?
# Packages without any description text cannot be embedded at all -
# exclude them so they do not fail on every single run.
base_scope = Package.where("COALESCE(description, '') <> '' OR " \
"COALESCE(long_description, '') <> ''")
scope = force ? base_scope : base_scope.where(embedding: nil)
total = scope.count
if total.zero?
logger.info 'No packages need vector computation.'
next
end
logger.info "Computing vectors for #{total} packages " \
"(#{force ? 'recomputing' : 'missing only'}, " \
"service: #{Vectorizer.service_url})"
done = 0
failed = 0
# Remember the most recent SQL statement so failures can be traced
# to the exact query that triggered them.
last_sql = nil
ActiveSupport::Notifications.subscribe('sql.active_record') do |*args|
last_sql = args.last[:sql]
end
scope.find_each do |package|
begin
raise ArgumentError, 'no description text' if package.embedding_text.blank?
package.update_embedding!
done += 1
rescue StandardError => e
# A single broken package must never stop the whole run - log
# enough detail to diagnose it and move on.
failed += 1
logger.error "Failed for package #{package.name}: #{e.class}: #{e.message}"
logger.error "Last SQL: #{last_sql}"
logger.error e.backtrace.first(15).join("\n") if e.backtrace
end
processed = done + failed
next unless (processed % 100).zero? || processed == total
percent = (processed.to_f / total * 100).round(1)
logger.info "Progress: #{processed}/#{total} (#{percent}%)"
end
logger.info "Done. #{done} vectors computed, #{failed} failures."
end
end
# rubocop:enable Metrics/BlockLength

View file

@ -1,70 +0,0 @@
# frozen_string_literal: true
require 'net/http'
require 'json'
# Client for the external web service that turns text into embedding
# vectors for nearest neighbor search (pgvector).
#
# Vectorizer.embed('A fast text editor') # => [0.123, -0.456, ...]
#
# The service URL can be overridden with the VECTOR_SERVICE_URL
# environment variable.
class Vectorizer
# Raised when the vector service cannot be reached or returns garbage
class Error < StandardError; end
DEFAULT_URL = 'https://vector-search.workaround.org/vector'
class << self
# Compute the embedding vector for the given text.
# Returns an array of floats.
def embed(text)
new.embed(text)
end
def service_url
ENV.fetch('VECTOR_SERVICE_URL', DEFAULT_URL)
end
end
def embed(text)
response = post_text(text)
unless response.is_a?(Net::HTTPSuccess)
raise Error, "Vector service returned HTTP #{response.code}"
end
JSON.parse(response.body).fetch('vector')
rescue JSON::ParserError => e
raise Error, "Vector service returned invalid JSON: #{e.message}"
rescue KeyError
raise Error, 'Vector service response did not contain a "vector" field'
end
private
def post_text(text)
uri = URI.parse(self.class.service_url)
request = build_request(uri, text)
options = { use_ssl: uri.scheme == 'https', read_timeout: 30, open_timeout: 10 }
Net::HTTP.start(uri.host, uri.port, **options) do |http|
http.request(request)
end
rescue SocketError, Errno::ECONNREFUSED, Net::OpenTimeout, Net::ReadTimeout => e
raise Error, "Could not reach vector service: #{e.message}"
end
def build_request(uri, text)
request = Net::HTTP::Post.new(uri.request_uri)
request['Content-Type'] = 'application/json'
request.body = { text: text }.to_json
# Optional Bearer token if the service runs with API_SECRET set
secret = ENV['VECTOR_SERVICE_SECRET']
request['Authorization'] = "Bearer #{secret}" if secret.present?
request
end
end

View file

@ -1,5 +1,3 @@
# frozen_string_literal: true
require 'test_helper'
class PackagesControllerTest < ActionController::TestCase
@ -22,118 +20,6 @@ class PackagesControllerTest < ActionController::TestCase
assert_select 'div', 'A program to browse web sites.'
end
test 'exact package name search wins without calling the vector service' do
with_stubbed_method(Package, :nearest_to_text,
->(_text) { flunk 'should not call vector search' }) do
get :grid, params: { search: 'vim', show_sources: '1' }
assert_response :success
assert_select 'div', 'vim'
assert_equal({ packages(:vim).id => :exact }, assigns(:search_sources))
assert_select '.search-source .label', 'exact match'
end
end
test 'gibberish search shows no results and never calls the vector service' do
with_stubbed_method(Package, :nearest_to_text,
->(_text) { flunk 'should not call vector search' }) do
get :grid, params: { search: 'asdfgh zzzz' }
assert_response :success
# the grid renders every package in the database when no search
# filter applies - so an empty result must not fall back to that
assert_select 'div', text: /package-\d+/, count: 0
assert_empty assigns(:search_sources)
end
end
test 'name prefix matches rank above semantic results' do
with_stubbed_method(Package, :nearest_to_text,
->(_text, **_opts) { Package.where(name: packages(:vim).name) }) do
get :grid, params: { search: 'package' }
assert_response :success
# fixtures create package-0..package-99 - prefix hits must render
assert_select 'div', 'package-0'
# ...and before any semantically found package
assert response.body.index('package-0') < response.body.index('vim')
sources = assigns(:search_sources)
assert_equal :name_prefix, sources[Package.find_by(name: 'package-0').id]
assert_equal :semantic, sources[packages(:vim).id]
end
end
test 'prefix matches and semantic results are deduplicated' do
with_stubbed_method(Package, :nearest_to_text,
->(_text, **_opts) { Package.where(name: packages(:firefox).name) }) do
get :grid, params: { search: 'fire' }
assert_response :success
assert_select 'div', text: 'firefox', count: 1
# the higher tier wins the attribution
assert_equal :name_prefix, assigns(:search_sources)[packages(:firefox).id]
end
end
test 'compound words in search match packages with joined names' do
# "fire fox" should find "firefox" via name substring matching
with_stubbed_method(Package, :nearest_to_text,
->(_text, **_opts) { Package.none }) do
get :grid, params: { search: 'fire fox', show_sources: '1' }
assert_response :success
assert_select 'div', text: 'firefox', count: 1
assert_equal({ packages(:firefox).id => :name_contains },
assigns(:search_sources))
assert_select '.search-source .label', 'name match'
end
end
test 'search source badges stay hidden without show_sources param' do
with_stubbed_method(Package, :nearest_to_text,
->(_text, **_opts) { Package.none }) do
get :grid, params: { search: 'fire fox' }
assert_response :success
assert_select '.search-source', count: 0
end
end
test 'details shows semantically related packages' do
packages(:vim).update_column(:embedding, Array.new(384, 0.25))
packages(:firefox).update_column(:embedding, Array.new(384, 0.75))
get :details, params: { name: 'vim' }
assert_response :success
assert_select '#related .pkgname', text: 'firefox'
end
test 'details without embedding renders no related section' do
get :details, params: { name: 'vim' }
assert_response :success
assert_select '#related', count: 0
end
test 'search uses semantic (vector) results' do
relation = Package.where(name: packages(:vim).name)
with_stubbed_method(Package, :nearest_to_text, ->(_text, **_opts) { relation }) do
get :grid, params: { search: 'editor' }
assert_response :success
assert_select 'div', 'vim'
assert_equal({ packages(:vim).id => :semantic }, assigns(:search_sources))
end
end
# There is deliberately no fallback for an empty vector search
# result: KNN always returns something as long as packages have
# embeddings. Full-text search is only used when the vector service
# cannot be reached.
test 'search falls back to full-text search when vector service fails' do
with_stubbed_method(Vectorizer, :embed,
->(_text) { raise Vectorizer::Error, 'service down' }) do
get :grid, params: { search: 'editor' }
assert_response :success
assert_select 'div', 'vim'
assert flash[:error].present?
assert_equal({ packages(:vim).id => :fulltext }, assigns(:search_sources))
end
end
# test 'should get thumbnail for a package and a desired version' do
# # get '/thumbnail-with-version/package-5/0.1'
# get thumbnail_with_version_url('package-5', '0.1')

View file

@ -1,107 +1,7 @@
# frozen_string_literal: true
require 'test_helper'
class PackageTest < ActiveSupport::TestCase
test 'embedding_text combines description and long description' do
package = packages(:firefox)
assert_equal "A web browser\nA program to browse web sites.", package.embedding_text
end
test 'embedding_text works with missing long description' do
package = packages(:firefox)
package.long_description = nil
assert_equal 'A web browser', package.embedding_text
end
test 'update_embedding! stores the vector returned by the vector service' do
vector = Array.new(384) { |i| i / 384.0 }
with_stubbed_method(Vectorizer, :embed, ->(_text) { vector }) do
package = packages(:vim)
assert package.update_embedding!
# The vector type stores float32 so allow small rounding differences
package.reload.embedding.to_a.each_with_index do |actual, i|
assert_in_delta vector[i], actual, 1e-6
end
end
end
test 'update_embedding! does nothing without description text' do
package = packages(:firefox)
package.description = nil
package.long_description = nil
with_stubbed_method(Vectorizer, :embed,
->(_text) { flunk 'should not call the vector service' }) do
assert_not package.update_embedding!
end
end
test 'nearest_to_text returns none for blank text' do
assert_empty Package.nearest_to_text('')
assert_empty Package.nearest_to_text(nil)
end
test 'name_starts_with matches name prefixes case-insensitively' do
assert_includes Package.name_starts_with('fir').map(&:name), packages(:firefox).name
assert_not_includes Package.name_starts_with('vim').map(&:name), packages(:firefox).name
assert_empty Package.name_starts_with('')
assert_empty Package.name_starts_with(nil)
end
test 'name_contains_all finds compound words split by spaces' do
assert_includes Package.name_contains_all(%w[fire fox]).map(&:name),
packages(:firefox).name
# all tokens must be present in the name
assert_empty Package.name_contains_all(%w[firefox zzzz])
assert_empty Package.name_contains_all([])
# single character tokens would flood results - ignore them
assert_empty Package.name_contains_all(['f'])
end
test 'lexical_overlap? pins the english dictionary regardless of server config' do
# Regression: with default_text_search_config = german (as on some
# servers) 'browser' stems to 'brows' which never matches the
# english-stemmed index.
connection = ActiveRecord::Base.connection
original = connection.select_value('SHOW default_text_search_config')
connection.execute("SET default_text_search_config = 'german'")
assert Package.lexical_overlap?('browser')
ensure
connection&.execute("SET default_text_search_config = '#{original}'")
end
test 'lexical_overlap? detects real words vs gibberish' do
# words present in fixture package descriptions
assert Package.lexical_overlap?('browser')
assert Package.lexical_overlap?('email client for vim syntax')
assert Package.lexical_overlap?('the browser') # stopwords are ignored
# gibberish, numbers and empty strings share no vocabulary
assert_not Package.lexical_overlap?('asdfgh zzzz')
assert_not Package.lexical_overlap?('123 !!!')
assert_not Package.lexical_overlap?('the') # stopword-only query
assert_not Package.lexical_overlap?('')
assert_not Package.lexical_overlap?(nil)
end
test 'nearest_to_text embeds the query and searches' do
vector = Array.new(384, 0.25)
embed_called_with = nil
with_stubbed_method(Vectorizer, :embed,
lambda { |text|
embed_called_with = text
vector
}) do
received = []
results = with_stubbed_method(Package, :nearest_neighbors,
lambda { |_field, received_vector, **_opts|
received << received_vector
[packages(:vim)]
}) do
Package.nearest_to_text('some search text')
end
assert_equal [packages(:vim)], results
assert_equal 'some search text', embed_called_with
assert_equal [vector], received
end
end
# test "the truth" do
# assert true
# end
end

View file

@ -1,5 +1,3 @@
# frozen_string_literal: true
ENV['RAILS_ENV'] ||= 'test'
require File.expand_path('../config/environment', __dir__)
require 'rails/test_help'
@ -17,23 +15,6 @@ module ActiveSupport
class TestCase
# Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
fixtures :all
# Replace a class-level method with a fixed return value or lambda
# for the duration of the block. Used to keep tests away from
# external services like the vector embedding web service.
def with_stubbed_method(klass, method_name, value_or_lambda)
original = klass.method(method_name)
klass.define_singleton_method(method_name) do |*args, **kwargs|
if value_or_lambda.respond_to?(:call)
value_or_lambda.call(*args, **kwargs)
else
value_or_lambda
end
end
yield
ensure
klass.define_singleton_method(method_name, original)
end
end
end