debshots/lib/tasks/import_debian.rake

require 'deb_importer'

# The repository format is documented at:
# https://wiki.debian.org/RepositoryFormat

include DebImporter

# List of regular expressions. If the package name matches any
# of these then the package will not be imported.
BLACKLIST_NAME_PATTERN=[
  /-data$/,
  /-dev$/,
  /-doc$/,
  /-dbg$/,
  /-common$/,
  /-l10n($|-)/,
  /-locale-/
]

# List of regular expressions. If the package's section matches
# any of these then the package will not be imported.
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
BLACKLIST_SECTION_PATTERN = [
  /^debian-installer$/,
  %r{/?translations$},
  %r{/?debug$},
  %r{/?kernel$},
  %r{/?localization$},
  %r{/?oldlibs$},
  %r{/?libdevel$},
  %r{/?cli-mono$},
  # Libraries never make useful screenshots - and unlike a "lib*" name
  # pattern this does not hit applications like libreoffice (editors)
  # or librecad (graphics)
  %r{/?libs$},
]

# List of regular expressions. If the package's short description
# matches any of these then the package will not be imported.
# Transitional/dummy stub packages exist only to ease upgrades to a
# renamed or split package and never get a useful screenshot. Debian
# has no dedicated metadata field for this - the Developer's Reference
# (6.9.7) documents these exact phrases as the convention package
# maintainers use, and as what deborphan --guess-dummy looks for.
# Deliberately not matching a bare "dummy": too many real packages use
# that word for actual functionality (e.g. xserver-xorg-video-dummy,
# fence-agents-dummy, "double dummy solver" bridge card libraries).
BLACKLIST_DESCRIPTION_PATTERN = [
  /transitional/i,
  /dummy package/i,
  /dependency package/i,
]

# Whether to delete a blacklisted package from the database
REMOVE_BLACKLISTED_PACKAGE = true

# Whether to delete packages that were not found in any configured
# source during a full import run (e.g. packages Debian has dropped
# entirely, or ones from a suite/architecture no longer configured).
# Unlike REMOVE_BLACKLISTED_PACKAGE this is opt-in: a package looking
# "orphaned" can also mean a mirror had a transient fetch problem, so
# update_from_deb_repos refuses to act on it unless every configured
# component/architecture was actually fetched successfully this run.
REMOVE_ORPHANED_PACKAGES = false

namespace :debshots do
  desc "Import/update package database from configured DEB repositories"

  task :update_from_deb_repos => :environment do
    # Counters for added, updated or removed packages from the database
    stats_added = 0
    stats_updated = 0
    stats_removed = 0

    # Every package name encountered in any configured source this run,
    # used to find orphans afterwards. A Hash is used as a cheap set.
    seen_package_names = {}
    # Set to true if any component/architecture could not be fetched,
    # so we know not to trust seen_package_names for orphan removal.
    sources_incomplete = false

    repositories = Rails.configuration.package_sources
    Rails.logger = Logger.new(STDOUT)
    Rails.logger.level = Logger::INFO
    #Rails.logger.level = Logger::DEBUG

    Rails.logger.info "Importing Debian package information"

    repositories.each do |repository|
      wanted_architectures = repository[:architectures]
      Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
      release = DebImporter::Release.new(repository[:url])
      Rails.logger.info "> Supported architectures are: #{release.architectures}"
      if wanted_architectures
        Rails.logger.info "> We only want architectures: #{wanted_architectures}"
      end
      Rails.logger.info "> Supported components are: #{release.components}"

      # TODO: Remember what we imported to show it on the /about page

      release.components.split.each do |component|
        Rails.logger.info "> Component: #{component}"
        release.architectures.split.each do |architecture|
          if wanted_architectures and not wanted_architectures.include?(architecture)
            Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
            next
          end
          Rails.logger.info ">> Architecture: #{architecture}"
          # Check if this component and architecture is available on this mirror
          packages = release.packages(component, architecture)
          unless packages
            Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
            sources_incomplete = true
            next
          end

          Rails.logger.info "Packages file for #{component} on #{architecture} found."
          Rails.logger.debug "Got: #{packages}"
          packages.each do |package|
            Rails.logger.info "> Package: #{package[:Package]}"
            #Rails.logger.debug "Fetching package informaton from the database"
            seen_package_names[package[:Package]] = true

            if package_name_blacklisted?(package[:Package]) or
               package_section_blacklisted?(package[:Section]) or
               package_description_blacklisted?(package[:Description])
              # Should the package get removed from the database?
              if REMOVE_BLACKLISTED_PACKAGE
                db_package = Package.find_by name: package[:Package]
                if db_package
                  Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
                  db_package.destroy
                  stats_removed += 1
                end
              end

              next
            end # if blacklisted

            db_package = Package.find_by name: package[:Package]
            if db_package  # package exists in the database
              # check if the parsed package data has a newer version
              current_version = DebImporter::Version.new(db_package.version)
              new_version = DebImporter::Version.new(package[:Version])

              Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
              if new_version > current_version
                # update the package information from new data
                update_data(package, db_package)
                Rails.logger.info "Updating package information"
                stats_updated += 1
              end
            else # new package - create it in the database
              Rails.logger.debug "No such package in our database. Creating one."
              db_package = Package.new
              update_data(package, db_package)
              stats_added += 1
            end

            # unless db_package
            #   Rails.logger.debug "No such package in our database. Creating one."
            #   db_package = Package.new
            #   stats_added += 1
            # end

            # # Rails.logger.debug "Found in database: #{db_package}"
            # data_has_changed = update_data(package, db_package)
            # stats_updated += 1 if data_has_changed
          end # package.each

        end # architectures.each
      end # components.each

      Rails.logger.info "Done parsing #{repository[:url]} repository."
      Rails.logger.info "#{stats_added} packages added"
      Rails.logger.info "#{stats_updated} packages updated"
      Rails.logger.info "#{stats_removed} packages removed"
      Rails.logger.info "--------------------------------------------"
    end # repositories.each

    stats_removed += remove_orphaned_packages(seen_package_names, sources_incomplete)
    Rails.logger.info "#{stats_removed} packages removed in total"

  end # task

  desc 'Update long description from i18n file'

  task :update_longdescription_from_deb_repos => :environment do

    repositories = Rails.configuration.package_sources
    Rails.logger = Logger.new(STDOUT)
    Rails.logger.level = Logger::INFO
    # Rails.logger.level = Logger::DEBUG

    Rails.logger.info "Importing long description from Debian repository (i18n)"

    repositories.each do |repository|
      Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}"
      release = DebImporter::Release.new(repository[:url])
      Rails.logger.info "> Supported components are: #{release.components}"

      release.components.split.each do |component|
        Rails.logger.info "> Component: #{component}"
        release.i18n(component, 'en').each do |pkg|
          Rails.logger.debug "i18n information: #{pkg}"
          unless pkg[:'Description-en']
            Rails.logger.debug "No Description-en found in section."
            next
          end
          if package_name_blacklisted? pkg[:Package]
            next
          end
          # See if we have that package in the database
          if db_pkg = Package.find_by(name: pkg[:Package])
            Rails.logger.info "Updating long description for package #{db_pkg.name}"
            text = pkg[:'Description-en']
            _, long_description = text.split("\n", 2)
            db_pkg.long_description = long_description
            db_pkg.save
          else
            Rails.logger.debug "Package not in database. Nothing to update."
          end
        end # pkg.each
      end # components.each


      Rails.logger.info "Done updating long descriptions from translation file."
      Rails.logger.info "--------------------------------------------"
    end # repositories.each

  end # task

  # TODO: Some code duplication. Should be refactord.
  desc "List configured DEB repositories"

  task :list_deb_repos => :environment do
    repositories = Rails.configuration.package_sources
    Rails.logger = Logger.new(STDOUT)
    Rails.logger.level = Logger::INFO
    # Rails.logger.level = Logger::DEBUG

    Rails.logger.info "Listing configured DEB repositories"

    repositories.each do |repository|
      wanted_architectures = repository[:architectures]
      Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
      release = DebImporter::Release.new(repository[:url])
      Rails.logger.info "> Supported architectures are: #{release.architectures}"
      if wanted_architectures
        Rails.logger.info "> We only want architectures: #{wanted_architectures}"
      end
      Rails.logger.info "> Supported components are: #{release.components}"

      release.components.split.each do |component|
        Rails.logger.info "> Component: #{component}"
        release.architectures.split.each do |architecture|
          if wanted_architectures and not wanted_architectures.include?(architecture)
            Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
            next
          end

          Rails.logger.info ">> Architecture: #{architecture}"
          # Check if this component and architecture is available on this mirror
          packages = release.packages(component, architecture)
          if packages
            Rails.logger.info "#{packages.count} packages found."
          else
            Rails.logger.info "No packages."
          end
        end # architectures.each
      end # components.each

    end # repositories.each

  end # task

end # namespace

# Update the information about a package in the database
# package: Information about new package (parsed from remote source)
# db_package: Information that currently exists in the database
def update_data(package, db_package)
  Rails.logger.debug "New information: #{package.inspect}"

  # Rails.logger.info "New package version found. Updating details in database."
  db_package.version = Version.new(package[:Version]).upstream
  db_package.name = package[:Package] unless db_package.name # set the name for new packages
  db_package.description = package[:Description][0..79]
  db_package.homepage = package[:Homepage]
  db_package.section = package[:Section]
  db_package.origin = package[:Origin]
  package[:Maintainer]=~/^(.+) <(.+)?>/
  db_package.maintainer = $1
  db_package.maintainer_email = $2
  db_package.save
  Rails.logger.debug "Saved package data: #{db_package.inspect}"
end

# Check if a package name is blacklisted and should not be imported.
# This prevents importing boring software that likely does not get a useful screenshot.
def package_name_blacklisted?(name)
  BLACKLIST_NAME_PATTERN.each do |pattern|
    if name=~pattern
      Rails.logger.debug "  > Blacklisted by name ('#{name}' matches '#{pattern}'')"
      return true
    end
  end
  return false
end

# Check if a package's section is blacklisted and should not be imported.
# This prevents importing boring software that likely does not get a useful screenshot.
def package_section_blacklisted?(section)
  BLACKLIST_SECTION_PATTERN.each do |pattern|
    if section=~pattern
      Rails.logger.debug "  > Blacklisted by section ('#{section}' matches '#{pattern.to_s}')"
      return true
    end
  end
  return false
end

# Check if a package's short description marks it as a transitional or
# dummy stub package (e.g. "transitional package", "transitional dummy
# package for foo"). These never get a useful screenshot.
def package_description_blacklisted?(description)
  BLACKLIST_DESCRIPTION_PATTERN.each do |pattern|
    if description.to_s =~ pattern
      Rails.logger.debug "  > Blacklisted by description ('#{description}' matches '#{pattern}')"
      return true
    end
  end
  false
end

# Remove packages that were not encountered in any configured source
# during this import run. Refuses to act if any component/architecture
# failed to fetch (sources_incomplete) or if seen_names came back empty
# (both would otherwise risk wiping out packages that are still very
# much alive in Debian, just missed because of a network hiccup).
# Returns the number of packages actually removed.
def orphan_removal_blocked?(seen_names, sources_incomplete)
  if sources_incomplete
    Rails.logger.warn '  > Skipping: not all configured components/architectures could be ' \
                       'fetched this run. Re-run when all sources are reachable.'
    return true
  end

  if seen_names.empty?
    Rails.logger.warn '  > Skipping: no packages were seen this run at all - refusing to ' \
                       'treat that as "everything is orphaned".'
    return true
  end

  false
end

def find_orphaned_package_ids(seen_names)
  Package.pluck(:id, :name).each_with_object([]) do |(id, name), ids|
    ids << id unless seen_names.key?(name)
  end
end

def destroy_orphaned_packages(ids)
  removed = 0
  Package.where(id: ids).find_each do |package|
    Rails.logger.info "  > Removing orphaned package '#{package.name}' from database"
    package.destroy
    removed += 1
  end
  removed
end

def remove_orphaned_packages(seen_names, sources_incomplete)
  Rails.logger.info 'Checking for packages no longer found in any configured source'
  return 0 if orphan_removal_blocked?(seen_names, sources_incomplete)

  orphan_ids = find_orphaned_package_ids(seen_names)

  unless REMOVE_ORPHANED_PACKAGES
    Rails.logger.info "  > #{orphan_ids.size} packages are orphaned (dry run - set " \
                       'REMOVE_ORPHANED_PACKAGES to actually remove them)'
    return 0
  end

  destroy_orphaned_packages(orphan_ids)
end