278 lines
10 KiB
Ruby
278 lines
10 KiB
Ruby
require 'deb_importer'
|
|
|
|
# The repository format is documented at:
|
|
# https://wiki.debian.org/RepositoryFormat
|
|
|
|
include DebImporter
|
|
|
|
# List of regular expressions. If the package name matches any
|
|
# of these then the package will not be imported.
|
|
BLACKLIST_NAME_PATTERN=[
|
|
/-data$/,
|
|
/-dev$/,
|
|
/-doc$/,
|
|
/-dbg$/,
|
|
/-common$/,
|
|
/-l10n($|-)/,
|
|
/-locale-/
|
|
]
|
|
|
|
# List of regular expressions. If the package's section matches
|
|
# any of these then the package will not be imported.
|
|
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
|
|
BLACKLIST_SECTION_PATTERN = [
|
|
/^debian-installer$/,
|
|
%r{/?translations$},
|
|
%r{/?debug$},
|
|
%r{/?kernel$},
|
|
%r{/?localization$},
|
|
%r{/?oldlibs$},
|
|
%r{/?libdevel$},
|
|
%r{/?cli-mono$},
|
|
]
|
|
|
|
# Whether to delete a blacklisted package from the database
|
|
REMOVE_BLACKLISTED_PACKAGE = true
|
|
|
|
namespace :debshots do
|
|
desc "Import/update package database from configured DEB repositories"
|
|
|
|
task :update_from_deb_repos => :environment do
|
|
# Counters for added, updated or removed packages from the database
|
|
stats_added = 0
|
|
stats_updated = 0
|
|
stats_removed = 0
|
|
|
|
repositories = Rails.configuration.package_sources
|
|
Rails.logger = Logger.new(STDOUT)
|
|
Rails.logger.level = Logger::INFO
|
|
#Rails.logger.level = Logger::DEBUG
|
|
|
|
Rails.logger.info "Importing Debian package information"
|
|
|
|
repositories.each do |repository|
|
|
wanted_architectures = repository[:architectures]
|
|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
|
|
release = DebImporter::Release.new(repository[:url])
|
|
Rails.logger.info "> Supported architectures are: #{release.architectures}"
|
|
if wanted_architectures
|
|
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
|
|
end
|
|
Rails.logger.info "> Supported components are: #{release.components}"
|
|
|
|
# TODO: Remember what we imported to show it on the /about page
|
|
|
|
release.components.split.each do |component|
|
|
Rails.logger.info "> Component: #{component}"
|
|
release.architectures.split.each do |architecture|
|
|
if wanted_architectures and not wanted_architectures.include?(architecture)
|
|
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
|
|
next
|
|
end
|
|
Rails.logger.info ">> Architecture: #{architecture}"
|
|
# Check if this component and architecture is available on this mirror
|
|
packages = release.packages(component, architecture)
|
|
unless packages
|
|
Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
|
|
next
|
|
end
|
|
|
|
Rails.logger.info "Packages file for #{component} on #{architecture} found."
|
|
Rails.logger.debug "Got: #{packages}"
|
|
packages.each do |package|
|
|
Rails.logger.info "> Package: #{package[:Package]}"
|
|
#Rails.logger.debug "Fetching package informaton from the database"
|
|
|
|
if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section]
|
|
# Should the package get removed from the database?
|
|
if REMOVE_BLACKLISTED_PACKAGE
|
|
db_package = Package.find_by name: package[:Package]
|
|
if db_package
|
|
Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
|
|
db_package.destroy
|
|
stats_removed += 1
|
|
end
|
|
end
|
|
|
|
next
|
|
end # if blacklisted
|
|
|
|
db_package = Package.find_by name: package[:Package]
|
|
if db_package # package exists in the database
|
|
# check if the parsed package data has a newer version
|
|
current_version = DebImporter::Version.new(db_package.version)
|
|
new_version = DebImporter::Version.new(package[:Version])
|
|
|
|
Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}"
|
|
if new_version.upstream > current_version.upstream
|
|
# update the package information from new data
|
|
update_data(package, db_package)
|
|
Rails.logger.info "Updating package information"
|
|
stats_updated += 1
|
|
end
|
|
else # new package - create it in the database
|
|
Rails.logger.debug "No such package in our database. Creating one."
|
|
db_package = Package.new
|
|
update_data(package, db_package)
|
|
stats_added += 1
|
|
end
|
|
|
|
# unless db_package
|
|
# Rails.logger.debug "No such package in our database. Creating one."
|
|
# db_package = Package.new
|
|
# stats_added += 1
|
|
# end
|
|
|
|
# # Rails.logger.debug "Found in database: #{db_package}"
|
|
# data_has_changed = update_data(package, db_package)
|
|
# stats_updated += 1 if data_has_changed
|
|
end # package.each
|
|
|
|
end # architectures.each
|
|
end # components.each
|
|
|
|
Rails.logger.info "Done parsing #{repository[:url]} repository."
|
|
Rails.logger.info "#{stats_added} packages added"
|
|
Rails.logger.info "#{stats_updated} packages updated"
|
|
Rails.logger.info "#{stats_removed} packages removed"
|
|
Rails.logger.info "--------------------------------------------"
|
|
end # repositories.each
|
|
|
|
end # task
|
|
|
|
desc 'Update long description from i18n file'
|
|
|
|
task :update_longdescription_from_deb_repos => :environment do
|
|
|
|
repositories = Rails.configuration.package_sources
|
|
Rails.logger = Logger.new(STDOUT)
|
|
Rails.logger.level = Logger::INFO
|
|
# Rails.logger.level = Logger::DEBUG
|
|
|
|
Rails.logger.info "Importing long description from Debian repository (i18n)"
|
|
|
|
repositories.each do |repository|
|
|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}"
|
|
release = DebImporter::Release.new(repository[:url])
|
|
Rails.logger.info "> Supported components are: #{release.components}"
|
|
|
|
release.components.split.each do |component|
|
|
Rails.logger.info "> Component: #{component}"
|
|
release.i18n(component, 'en').each do |pkg|
|
|
Rails.logger.debug "i18n information: #{pkg}"
|
|
unless pkg[:'Description-en']
|
|
Rails.logger.debug "No Description-en found in section."
|
|
next
|
|
end
|
|
if package_name_blacklisted? pkg[:Package]
|
|
next
|
|
end
|
|
# See if we have that package in the database
|
|
if db_pkg = Package.find_by(name: pkg[:Package])
|
|
Rails.logger.info "Updating long description for package #{db_pkg.name}"
|
|
text = pkg[:'Description-en']
|
|
_, long_description = text.split("\n", 2)
|
|
db_pkg.long_description = long_description
|
|
db_pkg.save
|
|
else
|
|
Rails.logger.debug "Package not in database. Nothing to update."
|
|
end
|
|
end # pkg.each
|
|
end # components.each
|
|
|
|
|
|
Rails.logger.info "Done updating long descriptions from translation file."
|
|
Rails.logger.info "--------------------------------------------"
|
|
end # repositories.each
|
|
|
|
end # task
|
|
|
|
# TODO: Some code duplication. Should be refactord.
|
|
desc "List configured DEB repositories"
|
|
|
|
task :list_deb_repos => :environment do
|
|
repositories = Rails.configuration.package_sources
|
|
Rails.logger = Logger.new(STDOUT)
|
|
Rails.logger.level = Logger::INFO
|
|
# Rails.logger.level = Logger::DEBUG
|
|
|
|
Rails.logger.info "Listing configured DEB repositories"
|
|
|
|
repositories.each do |repository|
|
|
wanted_architectures = repository[:architectures]
|
|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
|
|
release = DebImporter::Release.new(repository[:url])
|
|
Rails.logger.info "> Supported architectures are: #{release.architectures}"
|
|
if wanted_architectures
|
|
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
|
|
end
|
|
Rails.logger.info "> Supported components are: #{release.components}"
|
|
|
|
release.components.split.each do |component|
|
|
Rails.logger.info "> Component: #{component}"
|
|
release.architectures.split.each do |architecture|
|
|
if wanted_architectures and not wanted_architectures.include?(architecture)
|
|
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
|
|
next
|
|
end
|
|
|
|
Rails.logger.info ">> Architecture: #{architecture}"
|
|
# Check if this component and architecture is available on this mirror
|
|
packages = release.packages(component, architecture)
|
|
if packages
|
|
Rails.logger.info "#{packages.count} packages found."
|
|
else
|
|
Rails.logger.info "No packages."
|
|
end
|
|
end # architectures.each
|
|
end # components.each
|
|
|
|
end # repositories.each
|
|
|
|
end # task
|
|
|
|
end # namespace
|
|
|
|
# Update the information about a package in the database
|
|
# package: Information about new package (parsed from remote source)
|
|
# db_package: Information that currently exists in the database
|
|
def update_data(package, db_package)
|
|
Rails.logger.debug "New information: #{package.inspect}"
|
|
|
|
# Rails.logger.info "New package version found. Updating details in database."
|
|
db_package.version = Version.new(package[:Version]).upstream
|
|
db_package.name = package[:Package] unless db_package.name # set the name for new packages
|
|
db_package.description = package[:Description][0..79]
|
|
db_package.homepage = package[:Homepage]
|
|
db_package.section = package[:Section]
|
|
db_package.origin = package[:Origin]
|
|
package[:Maintainer]=~/^(.+) <(.+)?>/
|
|
db_package.maintainer = $1
|
|
db_package.maintainer_email = $2
|
|
db_package.save
|
|
Rails.logger.debug "Saved package data: #{db_package.inspect}"
|
|
end
|
|
|
|
# Check if a package name is blacklisted and should not be imported.
|
|
# This prevents importing boring software that likely does not get a useful screenshot.
|
|
def package_name_blacklisted?(name)
|
|
BLACKLIST_NAME_PATTERN.each do |pattern|
|
|
if name=~pattern
|
|
Rails.logger.debug " > Blacklisted by name ('#{name}' matches '#{pattern}'')"
|
|
return true
|
|
end
|
|
end
|
|
return false
|
|
end
|
|
|
|
# Check if a package's section is blacklisted and should not be imported.
|
|
# This prevents importing boring software that likely does not get a useful screenshot.
|
|
def package_section_blacklisted?(section)
|
|
BLACKLIST_SECTION_PATTERN.each do |pattern|
|
|
if section=~pattern
|
|
Rails.logger.debug " > Blacklisted by section ('#{section}' matches '#{pattern.to_s}')"
|
|
return true
|
|
end
|
|
end
|
|
return false
|
|
end
|