debshots/lib/deb_importer.rb
Christoph Haas c05b52895d Parsing of i18n file added.
Allows us to read long descriptions and add them to the package information
2015-04-23 00:29:30 +02:00

160 lines
5.4 KiB
Ruby

# Various helper methods to update the database of packages
require 'bzip2'
module DebImporter
# This module imports information about packages of a Linux distribution
# that uses the DEB package format like Debian, Ubuntu or Mint.
#
# It starts by loading the Release file of a release to get
# information about available components and architectures.
#
# Next it loads the Packages lists (prefers bz2, falls back to
# .gz or even the uncompressed version).
class Release
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
# Load and parse a Release file of an APT repository
def initialize(dist_url, components)
@dist_url = dist_url
release_url = dist_url + "/Release"
Rails.logger.debug "Loading Release file from #{release_url}"
open(release_url) do |release_data|
fields = get_fields(release_data)
@architectures = fields[:Architectures]
@components = fields[:Components]
@description = fields[:Description]
@codename = fields[:Codename]
@origin = fields[:Origin]
@version = fields[:Version]
# Not needed at the moment. We do not check sizes or checksums yet.
#when 'SHA1', 'SHA256', 'MD5Sum'
# next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them
# @files = []
# value.lines.each do |line|
# path = line.split.last
# @files << path
# end
end # open
end # def initialize
# TODO: Avoid code duplication (bz2, gz, ...)
def i18n(component, language)
for suffix in ['.bz2', '.gz', '']
url = "#{@dist_url}/#{component}/i18n/Translation-en#{suffix}"
Rails.logger.debug "Looking for translations file: #{url}"
begin
Rails.logger.debug "Try opening URL: #{url}"
begin
file = open(url)
rescue OpenURI::HTTPError => e
Rails.logger.info "URL #{url} lead to #{e}. skipping."
next
end
if suffix == '.gz'
file = Zlib::GzipReader.new(file)
elsif suffix == '.bz2'
file = Bzip2::Reader.new(file)
end
Rails.logger.debug "File containing translations is: #{file}"
return get_paragraphs(file)
rescue Errno::ENOENT
Rails.logger.debug "URL could not be opened. Skipping."
end
end
end
# Try to load the Packages file for a certain component (e.g. "main")
# and architecture (e.g. "amd64")
def packages(component, architecture)
# create path like "main/binary-amd64/Packages"
packages_path = "/#{component}/binary-#{architecture}/Packages"
# Check if gzip or uncompressed files exist
# (there is no working bzip2 library for Ruby 2.x at the time - 11/2014)
for suffix in ['.bz2', '.gz', '']
packages_path_with_suffix = packages_path+suffix
Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}"
url = @dist_url + packages_path_with_suffix
begin
Rails.logger.debug "Try opening URL: #{url}"
begin
file = open(url)
rescue OpenURI::HTTPError => e
Rails.logger.info "URL #{url} lead to #{e}. skipping."
next
end
if suffix == '.gz'
file = Zlib::GzipReader.new(file)
elsif suffix == '.bz2'
file = Bzip2::Reader.new(file)
end
Rails.logger.debug "File containing packages is: #{file}"
#paragraphs = get_paragraphs(file)
#return paragraphs
#byebug
return get_paragraphs(file)
rescue Errno::ENOENT
Rails.logger.debug "URL could not be opened. Skipping."
end
end
return nil
end # def packages
end # class Release
private
# Gather the fields of a Debian control file and return them as a hash
def get_fields(data)
fields = {}
name=value=''
data.each_line do |line|
case line
when /^(.+): (.+)/ # "Key: Value"
fields[name.to_sym]=value unless value.empty?
name,value=$1,$2
when /^(.+):\s*/ # "Key:" (start of multi-line entry)
fields[name.to_sym]=value unless value.empty?
name=$1
value=''
when /^\s+(.+)/ # " Indented multi-line value"
value << $1+"\n"
when /^\s+$/ # Empty line
break
end
end
# Any lines left at the end of the input?
fields[name.to_sym]=value unless value.empty?
return fields
end
# Iterator that splits up the input of a debian control file
# by empty lines. For example Debian "Packages" files consist
# of one paragraph for each package listed in it.
def get_paragraphs(data)
Enumerator.new do |enum|
gathered_lines = '' # collects all lines belonging to a field
data.each_line do |line|
if line.chomp.empty? # empty line found that seperates paragraphs
unless gathered_lines.empty? # any lines gathered so far?
enum.yield get_fields(gathered_lines)
gathered_lines = ''
end
else
gathered_lines << line
end
end # each_line
# Any lines left after the last empty line and the end of the input?
unless gathered_lines.empty?
enum.yield get_fields(gathered_lines)
end
end # Enumerator
end # def
end # module