debshots/lib/tasks/import_debian.rake

272 lines
9.8 KiB
Ruby

require 'open-uri' # allows to load URLs using open()
require 'deb_importer'
# The repository format is documented at:
# https://wiki.debian.org/RepositoryFormat
include DebImporter
# List of regular expressions. If the package name matches any
# of these then the package will not be imported.
BLACKLIST_NAME_PATTERN=[
/-data$/,
/-dev$/,
/-doc$/,
/-dbg$/,
/-common$/,
/-l10n-/,
/-locale-/,
]
# List of regular expressions. If the package's section matches
# any of these then the package will not be imported.
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
BLACKLIST_SECTION_PATTERN = [
/^debian-installer$/,
/\/?text$/,
/\/?translations$/,
/\/?debug$/,
/\/?kernel$/,
/\/?libs$/,
/\/?localization$/,
/\/?oldlibs$/,
/\/?otherosfs$/,
/\/?doc$/,
/\/?libdevel$/,
/\/?cli-mono$/,
]
# Whether to delete a blacklisted package from the database
REMOVE_BLACKLISTED_PACKAGE = true
namespace :debshots do
desc "Import/update package database from configured DEB repositories"
task :update_from_deb_repos => :environment do
# Counters for added, updated or removed packages from the database
stats_added = 0
stats_updated = 0
stats_removed = 0
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
#Rails.logger.level = Logger::DEBUG
Rails.logger.info "Importing Debian package information"
repositories.each do |repository|
wanted_architectures = repository[:architectures]
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported architectures are: #{release.architectures}"
if wanted_architectures
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
end
Rails.logger.info "> Supported components are: #{release.components}"
# TODO: Remember what we imported to show it on the /about page
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.architectures.split.each do |architecture|
if wanted_architectures and not wanted_architectures.include?(architecture)
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
next
end
Rails.logger.info ">> Architecture: #{architecture}"
# Check if this component and architecture is available on this mirror
packages = release.packages(component, architecture)
unless packages
Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
next
end
Rails.logger.info "Packages file for #{component} on #{architecture} found."
Rails.logger.debug "Got: #{packages}"
packages.each do |package|
Rails.logger.info "> Package: #{package[:Package]}"
#Rails.logger.debug "Fetching package informaton from the database"
if package_blacklisted? package
# Should the package get removed from the database?
if REMOVE_BLACKLISTED_PACKAGE
db_package = Package.find_by name: package[:Package]
if db_package
Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
db_package.destroy
stats_removed +=1
end
end
next
end # if blacklisted
db_package = Package.find_by name: package[:Package]
unless db_package
Rails.logger.debug "No such package in our database. Creating one."
db_package = Package.create
stats_added += 1
end
# Rails.logger.debug "Found in database: #{db_package}"
update_data(package, db_package)
stats_updated += 1
end # package.each
end # architectures.each
end # components.each
Rails.logger.info "Done parsing #{repository[:url]} repository."
Rails.logger.info "#{stats_added} packages added"
Rails.logger.info "#{stats_updated} packages updated"
Rails.logger.info "#{stats_removed} packages removed"
Rails.logger.info "--------------------------------------------"
end # repositories.each
end # task
desc 'Update long description from i18n file'
task :update_longdescription_from_deb_repos => :environment do
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
Rails.logger.level = Logger::DEBUG
Rails.logger.info "Importing long descriptoin from Debian repository (i18n)"
# BUG
# Description-en: autoconf like tool
# ACR is an autoconf like tool that allows you to create configure scripts for
# your programs. The main aim of this tool is to teach developers how to create
# portable builds of their tools, just using generic functions wrapped by acr to
# generate portable shellscript.
# Parses as "your programs. The ..."
repositories.each do |repository|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported components are: #{release.components}"
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.i18n(component, 'en').each do |pkg|
Rails.logger.debug "i18n information: #{pkg}"
next unless pkg[:'Description-en']
# See if we have that package in the database
if db_pkg = Package.find_by(name: pkg[:Package])
Rails.logger.info "Updating long description for package #{db_pkg.name}"
text = pkg[:'Description-en']
short_description, long_description = text.split("\n", 2)
db_pkg.long_description = long_description
db_pkg.save
end
end # pkg.each
end # components.each
Rails.logger.info "Done updating long descriptions from translation file."
Rails.logger.info "--------------------------------------------"
end # repositories.each
end # task
# TODO: Some code duplication. Should be refactord.
desc "List configured DEB repositories"
task :list_deb_repos => :environment do
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
# Rails.logger.level = Logger::INFO
Rails.logger.level = Logger::DEBUG
Rails.logger.info "Listing configured DEB repositories"
repositories.each do |repository|
wanted_architectures = repository[:architectures]
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported architectures are: #{release.architectures}"
if wanted_architectures
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
end
Rails.logger.info "> Supported components are: #{release.components}"
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.architectures.split.each do |architecture|
if wanted_architectures and not wanted_architectures.include?(architecture)
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
next
end
Rails.logger.info ">> Architecture: #{architecture}"
# Check if this component and architecture is available on this mirror
packages = release.packages(component, architecture)
if packages
Rails.logger.info "#{packages.count} packages found."
else
Rails.logger.info "No packages."
end
end # architectures.each
end # components.each
end # repositories.each
end # task
end # namespace
# Update the information about a package in the database
def update_data(package, db_package)
Rails.logger.debug "New information: #{package.to_yaml}"
Rails.logger.debug "Database information: #{db_package.to_yaml}"
# Check if the version changed
# Only the actual upstream software version (e.g. 1.2) but not the package
# revision (e.g. 4:1.2-ubuntu0)
package[:Version] =~ /(?:\d+\:)?(.+?)(~|\+|\-|$)/
new_version = $1
if not new_version
Rails.logger.fatal "Could not parse version string: #{package[:Version]}"
end
if new_version != package[:Version]
Rails.logger.debug "- Version changed: #{db_package.version} -> #{new_version}"
db_package.version = new_version
end
db_package.save
# TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
end
# Check if a package is blacklisted and should not be imported.
# This prevents importing boring software that likely does not get a useful screenshot.
# Argument: Hash containing package information from a Packages file
def package_blacklisted?(package)
Rails.logger.debug "Blacklist check of package #{package[:Package]}"
# TODO: add a flag that allows to manually blacklist a package
BLACKLIST_SECTION_PATTERN.each do |pattern|
Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}"
if package[:Section] =~ pattern
Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')"
return true
end
end
BLACKLIST_NAME_PATTERN.each do |pattern|
Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}"
if package[:Package] =~ pattern
Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')"
return true
end
end
false
end