debshots/lib/deb_importer.rb

223 lines
7.7 KiB
Ruby

# Various helper methods to update the database of packages
require 'bzip2'
module DebImporter
# This module imports information about packages of a Linux distribution
# that uses the DEB package format like Debian, Ubuntu or Mint.
#
# It starts by loading the Release file of a release to get
# information about available components and architectures.
#
# Next it loads the Packages lists (prefers bz2, falls back to
# .gz or even the uncompressed version).
class Release
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
# Load and parse a Release file of an APT repository
def initialize(dist_url)
@dist_url = dist_url
release_url = dist_url + "/Release"
Rails.logger.debug "Loading Release file from #{release_url}"
open(release_url) do |release_data|
fields = get_fields(release_data)
@architectures = fields[:Architectures]
@components = fields[:Components]
@description = fields[:Description]
@codename = fields[:Codename]
@origin = fields[:Origin]
@version = fields[:Version]
# TODO: check sizes and checksums
end # open
end # def initialize
# Get package information from translation (i18n) files.
# Returns an enumerator of packages.
def i18n(component, language)
url = "#{@dist_url}/#{component}/i18n/Translation-en"
file = find_and_open_compressed_url(url)
if file
return get_paragraphs(file)
else
return []
end
end
# Look for the file or URL in various compressed formats
# (e.g. bz2, gz) and fall back to plain text format.
def find_and_open_compressed_url(base_url)
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
for suffix in ['.bz2', '.gz', '']
begin
begin
url = "#{base_url}#{suffix}"
Rails.logger.debug "Checking if file at #{url} is available"
file = open(url)
rescue OpenURI::HTTPError => e
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
next
end
# Decompress file depending on its filename suffix
case suffix
when '.bz2'
file = Bzip2::Reader.new(file)
when '.gz'
file = Zlib::GzipReader.new(file)
end
Rails.logger.debug "File containing translations is: #{file}"
return file
rescue Errno::ENOENT
Rails.logger.debug "URL #{url} could not be opened. Skipping."
end
end
Rails.logger.error "No file found at #{url} and various compression extensions."
return nil
end
# Try to load the Packages file for a certain component (e.g. "main")
# and architecture (e.g. "amd64")
def packages(component, architecture)
# create path like "main/binary-amd64/Packages"
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
Rails.logger.debug "Loading packages from #{packages_path}"
file = find_and_open_compressed_url(packages_path)
return get_paragraphs(file)
end # def packages
end # class Release
private
# Gather the fields of a Debian control file section and return them as a hash
def get_fields(data)
fields = {}
name=value=''
data.each_line do |line|
case line
when /^(.+?): (.+)/ # "Key: Value"
fields[name.to_sym]=value unless value.empty?
name,value=$1,$2
when /^(.+?):$/ # "Key:" (start of multi-line entry without value in line)
fields[name.to_sym]=value unless value.empty?
name=$1
value=''
when /^\s+(.+)/ # " Indented multi-line value"
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
unless value.empty?
value << "\n"
end
value << $1
when /^\s+$/ # Empty line
break
end
end
# Any lines left at the end of the input?
fields[name.to_sym]=value unless value.empty?
return fields
end
# Iterator that splits up the input of a debian control file
# by empty lines. For example Debian "Packages" files consist
# of one paragraph for each package listed in it.
def get_paragraphs(data)
Enumerator.new do |enum|
gathered_lines = '' # collects all lines belonging to a field
data.each_line do |line|
if line.chomp.empty? # empty line found that seperates paragraphs
unless gathered_lines.empty? # any lines gathered so far?
enum.yield get_fields(gathered_lines)
gathered_lines = ''
end
else
gathered_lines << line
end
end # each_line
# Any lines left after the last empty line and the end of the input?
unless gathered_lines.empty?
enum.yield get_fields(gathered_lines)
end
end # Enumerator
end # def
class Version
attr_reader :epoch, :version, :revision, :version_string
def initialize(version_string)
@version_string = version_string
# Split into "[epoch:]version[-revision]"
unless /^(?<epoch>(\d+)\:)?(?<version>.+)(?<revision>\-[\+\.~]+)?/ =~ version_string
raise ArgumentError, "Cannot parse version string: #{version_string}"
end
@epoch = epoch ? epoch : 0
@version = version
@revision = revision if revision
end # /def
def to_s
@version_string
end
def >(other)
dpkg_compare_version(@version_string, other, 'gt')
end
def <(other)
dpkg_compare_version(@version_string, other, 'lt')
end
private
# Determine which version string denotes the newer package
# v1: version of the first package
# v2: version of the second package
# op: the comparison operator (default: 'gt' / greater-than)
#
# The algorithm is described in the Debian Policy at
# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
# The reference implementation is "dpkg --compare-versions …"
def dpkg_compare_version(v1, v2, op)
system("dpkg --compare-versions #{v1} #{op} #{v2}")
end
# def ___version_compare(x,y)
# # Compare a version string (like the upstream_version or
# # debian_revision string) against another version string.
# # The algorithm works like this:
# #
# # The strings are compared from left to right.
# # First the initial part of each string consisting entirely of non-digit
# # characters is determined. These two parts (one of which may be empty)
# # are compared lexically. If a difference is found it is returned.
# # The lexical comparison is a comparison of ASCII values modified so
# # that all the letters sort earlier than all the non-letters and so that
# # a tilde sorts before anything, even the end of a part.
# # For example, the following parts are in sorted order from
# # earliest to latest: ~~, ~~a, ~, the empty part, a.[37]
# #
# # Then the initial part of the remainder of each string which
# # consists entirely of digit characters is determined. The
# # numerical values of these two parts are compared, and any
# # difference found is returned as the result of the comparison.
# # For these purposes an empty string (which can only occur at
# # the end of one or both version strings being compared) counts as zero.
# #
# # These two steps (comparing and removing initial non-digit strings
# # and initial digit strings) are repeated until a difference is
# # found or both strings are exhausted.
# (x.chars).zip(y.chars) do |xchar,ychar|
# puts xchar, ychar
# end
# end
end # /class
end # module