diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 05731a8..d8eb298 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -1,20 +1,51 @@ require 'open-uri' # allows to load URLs using open() require 'deb_importer' +# The repository format is documented at: +# https://wiki.debian.org/RepositoryFormat + include DebImporter +# List of regular expressions. If the package name matches any +# of these then the package will not be imported. BLACKLIST_NAME_PATTERN=[ - '-data$', - '-dev$', - '-doc$', + /-data$/, + /-dev$/, + /-doc$/, + /-dbg$/, + /-common$/, + /-l10n-/, + /-locale-/, ] +# List of regular expressions. If the package's section matches +# any of these then the package will not be imported. +# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug +BLACKLIST_SECTION_PATTERN = [ + /^debian-installer$/, + /\/?text$/, + /\/?translations$/, + /\/?debug$/, + /\/?kernel$/, + /\/?libs$/, + /\/?localization$/, + /\/?oldlibs$/, + /\/?otherosfs$/, + /\/?doc$/, + /\/?libdevel$/, + /\/?cli-mono$/, +] + +# Whether to delete a packag +REMOVE_BLACKLISTED_PACKAGE = true + desc "Import package information from Debian-style Packages files" task :import_debian => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::DEBUG + Rails.logger.level = Logger::INFO + #Rails.logger.level = Logger::DEBUG puts "Importing Debian package information" @@ -41,9 +72,22 @@ task :import_debian => :environment do Rails.logger.debug "Got: #{packages}" packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" - #Rails.logger.debug "Fetching package informaton from our database" - #db_package = Package[name: package[:Package]] + #Rails.logger.debug "Fetching package informaton from the database" + db_package = Package.find_by name: package[:Package] + + if package_blacklisted? package + # Should the package get removed from the database? + if REMOVE_BLACKLISTED_PACKAGE + if db_package + Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" + db_package.destroy + end + end + + next + end # if blacklisted + unless db_package Rails.logger.debug "No such package in our database." next @@ -51,22 +95,19 @@ task :import_debian => :environment do Rails.logger.debug "Found in database: #{db_package.id}" update_data(package, db_package) - end + end # package.each - end - end - end + end # architectures.each + end # components.each -end + Rails.logger.info "Done parsing #{repository[:url]} repository." + Rails.logger.info "--------------------------------------------" + end # repositories.each + +end # task # Update the information about a package in the database def update_data(package, db_package) - # Check update criteria (blacklist, whitelist) - #for name, name in expression - #code - #end - - Rails.logger.debug "New information: #{package.to_yaml}" Rails.logger.debug "Database information: #{db_package.to_yaml}" @@ -74,7 +115,6 @@ def update_data(package, db_package) # Only the actual upstream software version (e.g. 1.2) but not the package # revision (e.g. 4:1.2-ubuntu0) package[:Version] =~ /(?:\d+\:)?(.+?)(~|\+|\-|$)/ - #new_version = package[:Version].split('-')[0] new_version = $1 if not new_version Rails.logger.fatal "Could not parse version string: #{package[:Version]}" @@ -87,7 +127,32 @@ def update_data(package, db_package) db_package.save - # TODO: Deal with different distributions differently - # TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en end + +# Check if a package is blacklisted and should not be imported. +# This prevents importing boring software that likely does not get a useful screenshot. +# Argument: Hash containing package information from a Packages file +def package_blacklisted?(package) + Rails.logger.debug "Blacklist check of package #{package[:Package]}" + + # TODO: add a flag that allows to manually blacklist a package + + BLACKLIST_SECTION_PATTERN.each do |pattern| + Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}" + if package[:Section] =~ pattern + Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')" + return true + end + end + + BLACKLIST_NAME_PATTERN.each do |pattern| + Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}" + if package[:Package] =~ pattern + Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')" + return true + end + end + + false +end