Remove packages not found in any configured source

Tracks every package name seen across all repositories/components/
architectures during update_from_deb_repos and, once the run
completes, destroys database packages that were not among them (e.g.
ones Debian has dropped entirely, or ones from a since-unconfigured
suite/architecture).

Deliberately opt-in (REMOVE_ORPHANED_PACKAGES = false by default),
unlike REMOVE_BLACKLISTED_PACKAGE: a package looking 'orphaned' can
also just mean a mirror had a transient fetch problem, so the removal
refuses to run at all if any component/architecture failed to fetch,
or if no packages were seen this run at all.

lib/tasks/import_debian.rake

@@ -54,6 +54,15 @@ BLACKLIST_DESCRIPTION_PATTERN = [
 # Whether to delete a blacklisted package from the database
 REMOVE_BLACKLISTED_PACKAGE = true
 
+# Whether to delete packages that were not found in any configured
+# source during a full import run (e.g. packages Debian has dropped
+# entirely, or ones from a suite/architecture no longer configured).
+# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking
+# "orphaned" can also mean a mirror had a transient fetch problem, so
+# update_from_deb_repos refuses to act on it unless every configured
+# component/architecture was actually fetched successfully this run.
+REMOVE_ORPHANED_PACKAGES = false
+
 namespace :debshots do
   desc "Import/update package database from configured DEB repositories"
 
@@ -63,6 +72,13 @@ namespace :debshots do
     stats_updated = 0
     stats_removed = 0
 
+    # Every package name encountered in any configured source this run,
+    # used to find orphans afterwards. A Hash is used as a cheap set.
+    seen_package_names = {}
+    # Set to true if any component/architecture could not be fetched,
+    # so we know not to trust seen_package_names for orphan removal.
+    sources_incomplete = false
+
     repositories = Rails.configuration.package_sources
     Rails.logger = Logger.new(STDOUT)
     Rails.logger.level = Logger::INFO
@@ -94,6 +110,7 @@ namespace :debshots do
           packages = release.packages(component, architecture)
           unless packages
             Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
+            sources_incomplete = true
             next
           end
 
@@ -102,6 +119,7 @@ namespace :debshots do
           packages.each do |package|
             Rails.logger.info "> Package: #{package[:Package]}"
             #Rails.logger.debug "Fetching package informaton from the database"
+            seen_package_names[package[:Package]] = true
 
             if package_name_blacklisted?(package[:Package]) or
                package_section_blacklisted?(package[:Section]) or
@@ -160,6 +178,9 @@ namespace :debshots do
       Rails.logger.info "--------------------------------------------"
     end # repositories.each
 
+    stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete)
+    Rails.logger.info "#{stats_removed} packages removed in total"
+
   end # task
 
   desc 'Update long description from i18n file'
@@ -311,3 +332,56 @@ def package_description_blacklisted?(description)
   end
   false
 end
+
+# Remove packages that were not encountered in any configured source
+# during this import run. Refuses to act if any component/architecture
+# failed to fetch (sources_incomplete) or if seen_names came back empty
+# (both would otherwise risk wiping out packages that are still very
+# much alive in Debian, just missed because of a network hiccup).
+# Returns the number of packages actually removed.
+def orphan_removal_blocked?(seen_names, sources_incomplete)
+  if sources_incomplete
+    Rails.logger.warn '  > Skipping: not all configured components/architectures could be ' \
+                       'fetched this run. Re-run when all sources are reachable.'
+    return true
+  end
+
+  if seen_names.empty?
+    Rails.logger.warn '  > Skipping: no packages were seen this run at all - refusing to ' \
+                       'treat that as "everything is orphaned".'
+    return true
+  end
+
+  false
+end
+
+def find_orphaned_package_ids(seen_names)
+  Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids|
+    ids << id unless seen_names.key?(name)
+  end
+end
+
+def destroy_orphaned_packages(ids)
+  removed = 0
+  Package.where(id: ids).find_each do |package|
+    Rails.logger.info "  > Removing orphaned package '#{package.name}' from database"
+    package.destroy
+    removed += 1
+  end
+  removed
+end
+
+def remove_orphaned_packages(seen_names, sources_incomplete)
+  Rails.logger.info 'Checking for packages no longer found in any configured source'
+  return 0 if orphan_removal_blocked?(seen_names, sources_incomplete)
+
+  orphan_ids = find_orphaned_package_ids(seen_names)
+
+  unless REMOVE_ORPHANED_PACKAGES
+    Rails.logger.info "  > #{orphan_ids.size} packages are orphaned (dry run - set " \
+                       'REMOVE_ORPHANED_PACKAGES to actually remove them)'
+    return 0
+  end
+
+  destroy_orphaned_packages(orphan_ids)
+end