Script tidied up
This commit is contained in:
parent
b70d2e6701
commit
c4390af52a
1 changed files with 85 additions and 20 deletions
|
|
@ -1,20 +1,51 @@
|
|||
require 'open-uri' # allows to load URLs using open()
|
||||
require 'deb_importer'
|
||||
|
||||
# The repository format is documented at:
|
||||
# https://wiki.debian.org/RepositoryFormat
|
||||
|
||||
include DebImporter
|
||||
|
||||
# List of regular expressions. If the package name matches any
|
||||
# of these then the package will not be imported.
|
||||
BLACKLIST_NAME_PATTERN=[
|
||||
'-data$',
|
||||
'-dev$',
|
||||
'-doc$',
|
||||
/-data$/,
|
||||
/-dev$/,
|
||||
/-doc$/,
|
||||
/-dbg$/,
|
||||
/-common$/,
|
||||
/-l10n-/,
|
||||
/-locale-/,
|
||||
]
|
||||
|
||||
# List of regular expressions. If the package's section matches
|
||||
# any of these then the package will not be imported.
|
||||
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
|
||||
BLACKLIST_SECTION_PATTERN = [
|
||||
/^debian-installer$/,
|
||||
/\/?text$/,
|
||||
/\/?translations$/,
|
||||
/\/?debug$/,
|
||||
/\/?kernel$/,
|
||||
/\/?libs$/,
|
||||
/\/?localization$/,
|
||||
/\/?oldlibs$/,
|
||||
/\/?otherosfs$/,
|
||||
/\/?doc$/,
|
||||
/\/?libdevel$/,
|
||||
/\/?cli-mono$/,
|
||||
]
|
||||
|
||||
# Whether to delete a packag
|
||||
REMOVE_BLACKLISTED_PACKAGE = true
|
||||
|
||||
desc "Import package information from Debian-style Packages files"
|
||||
|
||||
task :import_debian => :environment do
|
||||
repositories = Rails.configuration.package_sources
|
||||
Rails.logger = Logger.new(STDOUT)
|
||||
Rails.logger.level = Logger::DEBUG
|
||||
Rails.logger.level = Logger::INFO
|
||||
#Rails.logger.level = Logger::DEBUG
|
||||
|
||||
puts "Importing Debian package information"
|
||||
|
||||
|
|
@ -41,9 +72,22 @@ task :import_debian => :environment do
|
|||
Rails.logger.debug "Got: #{packages}"
|
||||
packages.each do |package|
|
||||
Rails.logger.info "> Package: #{package[:Package]}"
|
||||
#Rails.logger.debug "Fetching package informaton from our database"
|
||||
#db_package = Package[name: package[:Package]]
|
||||
#Rails.logger.debug "Fetching package informaton from the database"
|
||||
|
||||
db_package = Package.find_by name: package[:Package]
|
||||
|
||||
if package_blacklisted? package
|
||||
# Should the package get removed from the database?
|
||||
if REMOVE_BLACKLISTED_PACKAGE
|
||||
if db_package
|
||||
Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
|
||||
db_package.destroy
|
||||
end
|
||||
end
|
||||
|
||||
next
|
||||
end # if blacklisted
|
||||
|
||||
unless db_package
|
||||
Rails.logger.debug "No such package in our database."
|
||||
next
|
||||
|
|
@ -51,22 +95,19 @@ task :import_debian => :environment do
|
|||
|
||||
Rails.logger.debug "Found in database: #{db_package.id}"
|
||||
update_data(package, db_package)
|
||||
end
|
||||
end # package.each
|
||||
|
||||
end
|
||||
end
|
||||
end
|
||||
end # architectures.each
|
||||
end # components.each
|
||||
|
||||
end
|
||||
Rails.logger.info "Done parsing #{repository[:url]} repository."
|
||||
Rails.logger.info "--------------------------------------------"
|
||||
end # repositories.each
|
||||
|
||||
end # task
|
||||
|
||||
# Update the information about a package in the database
|
||||
def update_data(package, db_package)
|
||||
# Check update criteria (blacklist, whitelist)
|
||||
#for name, name in expression
|
||||
#code
|
||||
#end
|
||||
|
||||
|
||||
Rails.logger.debug "New information: #{package.to_yaml}"
|
||||
Rails.logger.debug "Database information: #{db_package.to_yaml}"
|
||||
|
||||
|
|
@ -74,7 +115,6 @@ def update_data(package, db_package)
|
|||
# Only the actual upstream software version (e.g. 1.2) but not the package
|
||||
# revision (e.g. 4:1.2-ubuntu0)
|
||||
package[:Version] =~ /(?:\d+\:)?(.+?)(~|\+|\-|$)/
|
||||
#new_version = package[:Version].split('-')[0]
|
||||
new_version = $1
|
||||
if not new_version
|
||||
Rails.logger.fatal "Could not parse version string: #{package[:Version]}"
|
||||
|
|
@ -87,7 +127,32 @@ def update_data(package, db_package)
|
|||
|
||||
db_package.save
|
||||
|
||||
# TODO: Deal with different distributions differently
|
||||
|
||||
# TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
|
||||
end
|
||||
|
||||
# Check if a package is blacklisted and should not be imported.
|
||||
# This prevents importing boring software that likely does not get a useful screenshot.
|
||||
# Argument: Hash containing package information from a Packages file
|
||||
def package_blacklisted?(package)
|
||||
Rails.logger.debug "Blacklist check of package #{package[:Package]}"
|
||||
|
||||
# TODO: add a flag that allows to manually blacklist a package
|
||||
|
||||
BLACKLIST_SECTION_PATTERN.each do |pattern|
|
||||
Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}"
|
||||
if package[:Section] =~ pattern
|
||||
Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')"
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
BLACKLIST_NAME_PATTERN.each do |pattern|
|
||||
Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}"
|
||||
if package[:Package] =~ pattern
|
||||
Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')"
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
false
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue