# Various helper methods to update the database of packages require 'bzip2' module DebImporter # This module imports information about packages of a Linux distribution # that uses the DEB package format like Debian, Ubuntu or Mint. # # It starts by loading the Release file of a release to get # information about available components and architectures. # # Next it loads the Packages lists (prefers bz2, falls back to # .gz or even the uncompressed version). class Release attr_reader :architectures, :components, :description, :codename, :origin, :version, :files # Load and parse a Release file of an APT repository def initialize(dist_url) @dist_url = dist_url release_url = dist_url + "/Release" Rails.logger.debug "Loading Release file from #{release_url}" open(release_url) do |release_data| fields = get_fields(release_data) @architectures = fields[:Architectures] @components = fields[:Components] @description = fields[:Description] @codename = fields[:Codename] @origin = fields[:Origin] @version = fields[:Version] # TODO: check sizes and checksums end # open end # def initialize # Get package information from translation (i18n) files. # Returns an enumerator of packages. def i18n(component, language) url = "#{@dist_url}/#{component}/i18n/Translation-en" file = find_and_open_compressed_url(url) if file return get_paragraphs(file) else return [] end end # Look for the file or URL in various compressed formats # (e.g. bz2, gz) and fall back to plain text format. def find_and_open_compressed_url(base_url) Rails.logger.debug "Looking for files at URL #{base_url} with different compressions" for suffix in ['.bz2', '.gz', ''] begin begin url = "#{base_url}#{suffix}" Rails.logger.debug "Checking if file at #{url} is available" file = open(url) rescue OpenURI::HTTPError => e Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." next end # Decompress file depending on its filename suffix case suffix when '.bz2' file = Bzip2::Reader.new(file) when '.gz' file = Zlib::GzipReader.new(file) end Rails.logger.debug "File containing translations is: #{file}" return file rescue Errno::ENOENT Rails.logger.debug "URL #{url} could not be opened. Skipping." end end Rails.logger.error "No file found at #{url} and various compression extensions." return nil end # Try to load the Packages file for a certain component (e.g. "main") # and architecture (e.g. "amd64") def packages(component, architecture) # create path like "main/binary-amd64/Packages" packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages" Rails.logger.debug "Loading packages from #{packages_path}" file = find_and_open_compressed_url(packages_path) return get_paragraphs(file) end # def packages end # class Release private # Gather the fields of a Debian control file section and return them as a hash def get_fields(data) fields = {} name=value='' data.each_line do |line| case line when /^(.+?): (.+)/ # "Key: Value" fields[name.to_sym]=value unless value.empty? name,value=$1,$2 when /^(.+?):$/ # "Key:" (start of multi-line entry without value in line) fields[name.to_sym]=value unless value.empty? name=$1 value='' when /^\s+(.+)/ # " Indented multi-line value" # Add a newline for multi-line entries ("Key: Value\n Foo\n Bar") unless value.empty? value << "\n" end value << $1 when /^\s+$/ # Empty line break end end # Any lines left at the end of the input? fields[name.to_sym]=value unless value.empty? return fields end # Iterator that splits up the input of a debian control file # by empty lines. For example Debian "Packages" files consist # of one paragraph for each package listed in it. def get_paragraphs(data) Enumerator.new do |enum| gathered_lines = '' # collects all lines belonging to a field data.each_line do |line| if line.chomp.empty? # empty line found that seperates paragraphs unless gathered_lines.empty? # any lines gathered so far? enum.yield get_fields(gathered_lines) gathered_lines = '' end else gathered_lines << line end end # each_line # Any lines left after the last empty line and the end of the input? unless gathered_lines.empty? enum.yield get_fields(gathered_lines) end end # Enumerator end # def class Version attr_reader :epoch, :version, :revision, :version_string def initialize(version_string) @version_string = version_string # Split into "[epoch:]version[-revision]" unless /^(?(\d+)\:)?(?.+)(?\-[\+\.~]+)?/ =~ version_string raise ArgumentError, "Cannot parse version string: #{version_string}" end @epoch = epoch ? epoch : 0 @version = version @revision = revision if revision end # /def def to_s @version_string end def >(other) dpkg_compare_version(@version_string, other, 'gt') end def <(other) dpkg_compare_version(@version_string, other, 'lt') end private # Determine which version string denotes the newer package # v1: version of the first package # v2: version of the second package # op: the comparison operator (default: 'gt' / greater-than) # # The algorithm is described in the Debian Policy at # https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version # The reference implementation is "dpkg --compare-versions …" def dpkg_compare_version(v1, v2, op) system("dpkg --compare-versions #{v1} #{op} #{v2}") end # def ___version_compare(x,y) # # Compare a version string (like the upstream_version or # # debian_revision string) against another version string. # # The algorithm works like this: # # # # The strings are compared from left to right. # # First the initial part of each string consisting entirely of non-digit # # characters is determined. These two parts (one of which may be empty) # # are compared lexically. If a difference is found it is returned. # # The lexical comparison is a comparison of ASCII values modified so # # that all the letters sort earlier than all the non-letters and so that # # a tilde sorts before anything, even the end of a part. # # For example, the following parts are in sorted order from # # earliest to latest: ~~, ~~a, ~, the empty part, a.[37] # # # # Then the initial part of the remainder of each string which # # consists entirely of digit characters is determined. The # # numerical values of these two parts are compared, and any # # difference found is returned as the result of the comparison. # # For these purposes an empty string (which can only occur at # # the end of one or both version strings being compared) counts as zero. # # # # These two steps (comparing and removing initial non-digit strings # # and initial digit strings) are repeated until a difference is # # found or both strings are exhausted. # (x.chars).zip(y.chars) do |xchar,ychar| # puts xchar, ychar # end # end end # /class end # module