debshots/lib/deb_importer.rb

200 lines
6.6 KiB
Ruby
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# frozen_string_literal: true
require 'open-uri'
require 'bzip2/ffi'
require 'pp'
# Various helper methods to update the database of packages
# This module imports information about packages of a Linux distribution
# that uses the DEB package format like Debian, Ubuntu or Mint.
#
# It starts by loading the Release file of a release to get
# information about available components and architectures.
#
# Next it loads the Packages lists (prefers bz2, falls back to
# .gz or even the uncompressed version).
module DebImporter
# Defines a Debian release
class Release
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
# Load and parse a Release file of an APT repository
def initialize(dist_url)
@dist_url = dist_url
release_url = "#{dist_url}/Release"
Rails.logger.debug "Loading Release file from #{release_url}"
URI.open(release_url) do |release_data|
fields = get_fields(release_data)
@architectures = fields[:Architectures]
@components = fields[:Components]
@description = fields[:Description]
@codename = fields[:Codename]
@origin = fields[:Origin]
@version = fields[:Version]
# TODO: check sizes and checksums
end
end
# Get package information from translation (i18n) files.
# Returns an enumerator of packages.
def i18n(component, language)
url = "#{@dist_url}/#{component}/i18n/Translation-#{language}"
file = find_and_open_compressed_url(url)
return get_paragraphs(file) if file
[]
end
# Look for the file or URL in various compressed formats
# (e.g. bz2, gz) and fall back to plain text format.
def find_and_open_compressed_url(base_url)
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
['.bz2', '.gz', ''].each do |suffix|
begin
url = "#{base_url}#{suffix}"
Rails.logger.debug "Checking if file at #{url} is available"
file = URI.open(url)
rescue OpenURI::HTTPError => e
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
next
end
file2 = nil
# Decompress file depending on its filename suffix
file2 = case suffix
when '.bz2'
Bzip2::FFI::Reader.read(file)
when '.gz'
Zlib::GzipReader.new(file)
else # plain text
file
end
Rails.logger.debug "File containing translations is: #{url}"
# Return an enumerator that iterates over lines of the file
return file2
rescue Errno::ENOENT
Rails.logger.debug "URL #{url} could not be opened. Skipping."
end
Rails.logger.error "No file found at #{url} and various compression extensions."
end
# Try to load the Packages file for a certain component (e.g. "main")
# and architecture (e.g. "amd64")
def packages(component, architecture)
# create path like "main/binary-amd64/Packages"
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
Rails.logger.debug "Loading packages from #{packages_path}"
file = find_and_open_compressed_url(packages_path)
get_paragraphs(file)
end
end
private
# Gather the fields of a Debian control file section and return them as a hash
def get_fields(data)
fields = {}
name = value = ''
data.each do |line|
case line
when /^(\S+?): (.+)/ # "Key: Value"
fields[name.to_sym] = value unless value.empty?
name = $1
value = $2
when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line)
fields[name.to_sym] = value unless value.empty?
name = $1
value = ''
when /^\s(.+)/ # " Indented multi-line value"
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
value += "\n" unless value.empty?
value += $1
when /^\s+$/ # Empty line
break
end
end
# Any lines left at the end of the input?
fields[name.to_sym] = value unless value.empty?
# pp fields
# puts '------------'
fields
end
# Iterator that splits up the input of a debian control file
# by empty lines. For example Debian "Packages" files consist
# of one paragraph for each package listed in it.
def get_paragraphs(data)
Rails.logger.debug("get_paragraphs data: #{data.class}")
Enumerator.new do |enum|
gathered_lines = [] # collects all lines belonging to a field
data.each_line do |line|
# Rails.logger.debug("___" + line)
if line.chomp.empty? # empty line found that seperates paragraphs
if gathered_lines.any? # any lines gathered so far?
# Rails.logger.debug('>>>>>>>>>>>>>>>>>>>>>> Gathered lines:')
# Rails.logger.debug(gathered_lines.inspect)
# Rails.logger.debug('<<<<<<<<<<<<<<<<<<<<<<')
enum.yield get_fields(gathered_lines)
gathered_lines = []
end
else
gathered_lines << line
end
end
# Any lines left after the last empty line and the end of the input?
enum.yield get_fields(gathered_lines) if gathered_lines.any?
end
end
# Represents the version of a Debian package
class Version
attr_reader :epoch, :upstream, :revision, :version_string
def initialize(version_string)
@version_string = version_string
# Split into "[epoch:]upstream[-revision]"
# See: https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
unless /^((?<epoch>\d+):)?(?<upstream>.+?)(-(?<revision>.+))?$/ =~ version_string
raise ArgumentError, "Cannot parse version string: #{version_string}"
end
@epoch = epoch || '0'
@upstream = upstream
@revision = revision if revision
end
def to_s
@version_string
end
def >(other)
dpkg_compare_version(@version_string, other, 'gt')
end
def <(other)
dpkg_compare_version(@version_string, other, 'lt')
end
private
# Determine which version string denotes the newer package
# v1: version of the first package
# v2: version of the second package
# op: the comparison operator (default: 'gt' / greater-than)
#
# The algorithm is described in the Debian Policy at
# https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-Version
# The reference implementation is "dpkg --compare-versions …"
def dpkg_compare_version(v1, v2, op)
system("dpkg --compare-versions #{v1} #{op} #{v2}")
end
end # /class
end # module