require 'open-uri' # allows to load URLs using open() require 'deb_importer' # The repository format is documented at: # https://wiki.debian.org/RepositoryFormat include DebImporter # List of regular expressions. If the package name matches any # of these then the package will not be imported. BLACKLIST_NAME_PATTERN=[ /-data$/, /-dev$/, /-doc$/, /-dbg$/, /-common$/, /-l10n-/, /-locale-/, ] # List of regular expressions. If the package's section matches # any of these then the package will not be imported. # /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug BLACKLIST_SECTION_PATTERN = [ /^debian-installer$/, /\/?text$/, /\/?translations$/, /\/?debug$/, /\/?kernel$/, /\/?libs$/, /\/?localization$/, /\/?oldlibs$/, /\/?otherosfs$/, /\/?doc$/, /\/?libdevel$/, /\/?cli-mono$/, ] # Whether to delete a packag REMOVE_BLACKLISTED_PACKAGE = true desc "Import package information from Debian-style Packages files" task :import_debian => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO #Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing Debian package information" repositories.each do |repository| Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url], repository[:components]) Rails.logger.info "> Supported architectures are: #{release.architectures}" Rails.logger.info "> Supported components are: #{release.components}" # TODO: Remember what we imported to show it on the /about page release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.architectures.split.each do |architecture| Rails.logger.info ">> Architecture: #{architecture}" # Check if this component and architecture is available on this mirror packages = release.packages(component, architecture) unless packages Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" next end Rails.logger.info "Packages file for #{component} on #{architecture} found." Rails.logger.debug "Got: #{packages}" packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" if package_blacklisted? package # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] if db_package Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" db_package.destroy end end next end # if blacklisted db_package = Package.find_by name: package[:Package] unless db_package Rails.logger.debug "No such package in our database." next end Rails.logger.debug "Found in database: #{db_package.id}" update_data(package, db_package) end # package.each end # architectures.each end # components.each Rails.logger.info "Done parsing #{repository[:url]} repository." Rails.logger.info "--------------------------------------------" end # repositories.each end # task # Update the information about a package in the database def update_data(package, db_package) Rails.logger.debug "New information: #{package.to_yaml}" Rails.logger.debug "Database information: #{db_package.to_yaml}" # Check if the version changed # Only the actual upstream software version (e.g. 1.2) but not the package # revision (e.g. 4:1.2-ubuntu0) package[:Version] =~ /(?:\d+\:)?(.+?)(~|\+|\-|$)/ new_version = $1 if not new_version Rails.logger.fatal "Could not parse version string: #{package[:Version]}" end if new_version != package[:Version] Rails.logger.debug "- Version changed: #{db_package.version} -> #{new_version}" db_package.version = new_version end db_package.save # TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en end # Check if a package is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. # Argument: Hash containing package information from a Packages file def package_blacklisted?(package) Rails.logger.debug "Blacklist check of package #{package[:Package]}" # TODO: add a flag that allows to manually blacklist a package BLACKLIST_SECTION_PATTERN.each do |pattern| Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}" if package[:Section] =~ pattern Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')" return true end end BLACKLIST_NAME_PATTERN.each do |pattern| Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}" if package[:Package] =~ pattern Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')" return true end end false end