require 'deb_importer' # The repository format is documented at: # https://wiki.debian.org/RepositoryFormat include DebImporter # List of regular expressions. If the package name matches any # of these then the package will not be imported. BLACKLIST_NAME_PATTERN=[ /-data$/, /-dev$/, /-doc$/, /-dbg$/, /-common$/, /-l10n($|-)/, /-locale-/ ] # List of regular expressions. If the package's section matches # any of these then the package will not be imported. # /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug BLACKLIST_SECTION_PATTERN = [ /^debian-installer$/, %r{/?translations$}, %r{/?debug$}, %r{/?kernel$}, %r{/?localization$}, %r{/?oldlibs$}, %r{/?libdevel$}, %r{/?cli-mono$}, ] # Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true namespace :debshots do desc "Import/update package database from configured DEB repositories" task :update_from_deb_repos => :environment do # Counters for added, updated or removed packages from the database stats_added = 0 stats_updated = 0 stats_removed = 0 repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) # Rails.logger.level = Logger::INFO Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing Debian package information" repositories.each do |repository| wanted_architectures = repository[:architectures] Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported architectures are: #{release.architectures}" if wanted_architectures Rails.logger.info "> We only want architectures: #{wanted_architectures}" end Rails.logger.info "> Supported components are: #{release.components}" # TODO: Remember what we imported to show it on the /about page release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.architectures.split.each do |architecture| if wanted_architectures and not wanted_architectures.include?(architecture) Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." next end Rails.logger.info ">> Architecture: #{architecture}" # Check if this component and architecture is available on this mirror packages = release.packages(component, architecture) unless packages Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" next end Rails.logger.info "Packages file for #{component} on #{architecture} found." Rails.logger.debug "Got: #{packages}" packages.each do |package| Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] if db_package Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" db_package.destroy stats_removed += 1 end end next end # if blacklisted db_package = Package.find_by name: package[:Package] if db_package # package exists in the database # check if the parsed package data has a newer version current_version = DebImporter::Version.new(db_package.version) new_version = DebImporter::Version.new(package[:Version]) Rails.logger.debug "Comparing package version: old=#{db_package.version} new=#{new_version}" if new_version.upstream > current_version.upstream # update the package information from new data update_data(package, db_package) Rails.logger.info "Updating package information" stats_updated += 1 end else # new package - create it in the database Rails.logger.debug "No such package in our database. Creating one." db_package = Package.new data_has_changed = update_data(package, db_package) stats_added += 1 end # unless db_package # Rails.logger.debug "No such package in our database. Creating one." # db_package = Package.new # stats_added += 1 # end # # Rails.logger.debug "Found in database: #{db_package}" # data_has_changed = update_data(package, db_package) # stats_updated += 1 if data_has_changed end # package.each end # architectures.each end # components.each Rails.logger.info "Done parsing #{repository[:url]} repository." Rails.logger.info "#{stats_added} packages added" Rails.logger.info "#{stats_updated} packages updated" Rails.logger.info "#{stats_removed} packages removed" Rails.logger.info "--------------------------------------------" end # repositories.each end # task desc 'Update long description from i18n file' task :update_longdescription_from_deb_repos => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing long description from Debian repository (i18n)" repositories.each do |repository| Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported components are: #{release.components}" release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.i18n(component, 'en').each do |pkg| Rails.logger.debug "i18n information: #{pkg}" unless pkg[:'Description-en'] Rails.logger.debug "No Description-en found in section." next end if package_name_blacklisted? pkg[:Package] next end # See if we have that package in the database if db_pkg = Package.find_by(name: pkg[:Package]) Rails.logger.info "Updating long description for package #{db_pkg.name}" text = pkg[:'Description-en'] short_description, long_description = text.split("\n", 2) db_pkg.long_description = long_description db_pkg.save else Rails.logger.debug "Package not in database. Nothing to update." end end # pkg.each end # components.each Rails.logger.info "Done updating long descriptions from translation file." Rails.logger.info "--------------------------------------------" end # repositories.each end # task # TODO: Some code duplication. Should be refactord. desc "List configured DEB repositories" task :list_deb_repos => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::DEBUG Rails.logger.info "Listing configured DEB repositories" repositories.each do |repository| wanted_architectures = repository[:architectures] Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported architectures are: #{release.architectures}" if wanted_architectures Rails.logger.info "> We only want architectures: #{wanted_architectures}" end Rails.logger.info "> Supported components are: #{release.components}" release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.architectures.split.each do |architecture| if wanted_architectures and not wanted_architectures.include?(architecture) Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." next end Rails.logger.info ">> Architecture: #{architecture}" # Check if this component and architecture is available on this mirror packages = release.packages(component, architecture) if packages Rails.logger.info "#{packages.count} packages found." else Rails.logger.info "No packages." end end # architectures.each end # components.each end # repositories.each end # task end # namespace # Update the information about a package in the database # package: Information about new package (parsed from remote source) # db_package: Information that currently exists in the database def update_data(package, db_package) Rails.logger.debug "New information: #{package.inspect}" # Rails.logger.info "New package version found. Updating details in database." db_package.version = Version.new(package[:Version]).upstream db_package.name = package[:Package] unless db_package.name # set the name for new packages db_package.description = package[:Description][0..79] db_package.homepage = package[:Homepage] db_package.section = package[:Section] db_package.origin = package[:Origin] package[:Maintainer]=~/^(.+) <(.+)?>/ db_package.maintainer = $1 db_package.maintainer_email = $2 db_package.save Rails.logger.debug "Saved package data: #{db_package.inspect}" end # Check if a package name is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. def package_name_blacklisted?(name) BLACKLIST_NAME_PATTERN.each do |pattern| if name=~pattern Rails.logger.debug " > Blacklisted by name ('#{name}' matches '#{pattern}'')" return true end end return false end # Check if a package's section is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. def package_section_blacklisted?(section) BLACKLIST_SECTION_PATTERN.each do |pattern| if section=~pattern Rails.logger.debug " > Blacklisted by section ('#{section}' matches '#{pattern.to_s}')" return true end end return false end