bzip2 import fixed
This commit is contained in:
parent
1095ae3aa5
commit
48e3a8c67f
6 changed files with 114 additions and 75 deletions
|
|
@ -1,25 +1,30 @@
|
|||
# Various helper methods to update the database of packages
|
||||
# frozen_string_literal: true
|
||||
|
||||
require 'open-uri'
|
||||
require 'bzip2/ffi'
|
||||
require 'pp'
|
||||
|
||||
# Various helper methods to update the database of packages
|
||||
# This module imports information about packages of a Linux distribution
|
||||
# that uses the DEB package format like Debian, Ubuntu or Mint.
|
||||
#
|
||||
# It starts by loading the Release file of a release to get
|
||||
# information about available components and architectures.
|
||||
#
|
||||
# Next it loads the Packages lists (prefers bz2, falls back to
|
||||
# .gz or even the uncompressed version).
|
||||
module DebImporter
|
||||
|
||||
# This module imports information about packages of a Linux distribution
|
||||
# that uses the DEB package format like Debian, Ubuntu or Mint.
|
||||
#
|
||||
# It starts by loading the Release file of a release to get
|
||||
# information about available components and architectures.
|
||||
#
|
||||
# Next it loads the Packages lists (prefers bz2, falls back to
|
||||
# .gz or even the uncompressed version).
|
||||
|
||||
# Defines a Debian release
|
||||
class Release
|
||||
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
|
||||
|
||||
# Load and parse a Release file of an APT repository
|
||||
def initialize(dist_url)
|
||||
def initialize(dist_url) # rubocop:disable Metrics/MethodLength
|
||||
@dist_url = dist_url
|
||||
release_url = dist_url + "/Release"
|
||||
release_url = "#{dist_url}/Release"
|
||||
Rails.logger.debug "Loading Release file from #{release_url}"
|
||||
open(release_url) do |release_data|
|
||||
URI.open(release_url) do |release_data|
|
||||
fields = get_fields(release_data)
|
||||
@architectures = fields[:Architectures]
|
||||
@components = fields[:Components]
|
||||
|
|
@ -29,13 +34,13 @@ module DebImporter
|
|||
@version = fields[:Version]
|
||||
|
||||
# TODO: check sizes and checksums
|
||||
end # open
|
||||
end # def initialize
|
||||
end
|
||||
end
|
||||
|
||||
# Get package information from translation (i18n) files.
|
||||
# Returns an enumerator of packages.
|
||||
def i18n(component, language)
|
||||
url = "#{@dist_url}/#{component}/i18n/Translation-en"
|
||||
url = "#{@dist_url}/#{component}/i18n/Translation-#{language}"
|
||||
file = find_and_open_compressed_url(url)
|
||||
if file
|
||||
return get_paragraphs(file)
|
||||
|
|
@ -48,34 +53,36 @@ module DebImporter
|
|||
# (e.g. bz2, gz) and fall back to plain text format.
|
||||
def find_and_open_compressed_url(base_url)
|
||||
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
|
||||
for suffix in ['.bz2', '.gz', '']
|
||||
['.bz2', '.gz', ''].each do |suffix|
|
||||
begin
|
||||
begin
|
||||
url = "#{base_url}#{suffix}"
|
||||
Rails.logger.debug "Checking if file at #{url} is available"
|
||||
file = open(url)
|
||||
rescue OpenURI::HTTPError => e
|
||||
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
|
||||
next
|
||||
end
|
||||
|
||||
# Decompress file depending on its filename suffix
|
||||
case suffix
|
||||
when '.bz2'
|
||||
file = Bzip2::Reader.new(file)
|
||||
when '.gz'
|
||||
file = Zlib::GzipReader.new(file)
|
||||
end
|
||||
Rails.logger.debug "File containing translations is: #{file}"
|
||||
|
||||
return file
|
||||
rescue Errno::ENOENT
|
||||
Rails.logger.debug "URL #{url} could not be opened. Skipping."
|
||||
url = "#{base_url}#{suffix}"
|
||||
Rails.logger.debug "Checking if file at #{url} is available"
|
||||
file = URI.open(url)
|
||||
rescue OpenURI::HTTPError => e
|
||||
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
|
||||
next
|
||||
end
|
||||
|
||||
file2 = nil
|
||||
|
||||
# Decompress file depending on its filename suffix
|
||||
case suffix
|
||||
when '.bz2'
|
||||
file2 = Bzip2::FFI::Reader.read(file)
|
||||
when '.gz'
|
||||
file2 = Zlib::GzipReader.new(file)
|
||||
else # plain text
|
||||
file2 = file
|
||||
end
|
||||
Rails.logger.debug "File containing translations is: #{url}"
|
||||
|
||||
# Return an enumerator that iterates over lines of the file
|
||||
return file2
|
||||
rescue Errno::ENOENT
|
||||
Rails.logger.debug "URL #{url} could not be opened. Skipping."
|
||||
end
|
||||
|
||||
Rails.logger.error "No file found at #{url} and various compression extensions."
|
||||
return nil
|
||||
end
|
||||
|
||||
# Try to load the Packages file for a certain component (e.g. "main")
|
||||
|
|
@ -85,9 +92,9 @@ module DebImporter
|
|||
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
|
||||
Rails.logger.debug "Loading packages from #{packages_path}"
|
||||
file = find_and_open_compressed_url(packages_path)
|
||||
return get_paragraphs(file)
|
||||
end # def packages
|
||||
end # class Release
|
||||
get_paragraphs(file)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
|
|
@ -95,22 +102,21 @@ module DebImporter
|
|||
def get_fields(data)
|
||||
fields = {}
|
||||
|
||||
name = value = ""
|
||||
data.each_line do |line|
|
||||
name = value = ''
|
||||
data.each do |line|
|
||||
case line
|
||||
when /^(\S+?): (.+)/ # "Key: Value"
|
||||
fields[name.to_sym] = value unless value.empty?
|
||||
name, value = $1, $2
|
||||
name = $1
|
||||
value = $2
|
||||
when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line)
|
||||
fields[name.to_sym] = value unless value.empty?
|
||||
name = $1
|
||||
value = ""
|
||||
when /^\s(.+)/ # " Indented multi-line value"
|
||||
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
|
||||
unless value.empty?
|
||||
value << "\n"
|
||||
end
|
||||
value << $1
|
||||
value += "\n" unless value.empty?
|
||||
value += $1
|
||||
when /^\s+$/ # Empty line
|
||||
break
|
||||
end
|
||||
|
|
@ -118,34 +124,40 @@ module DebImporter
|
|||
# Any lines left at the end of the input?
|
||||
fields[name.to_sym] = value unless value.empty?
|
||||
|
||||
return fields
|
||||
# pp fields
|
||||
# puts '------------'
|
||||
fields
|
||||
end
|
||||
|
||||
# Iterator that splits up the input of a debian control file
|
||||
# by empty lines. For example Debian "Packages" files consist
|
||||
# of one paragraph for each package listed in it.
|
||||
def get_paragraphs(data)
|
||||
Rails.logger.debug("get_paragraphs – data: #{data.class}")
|
||||
Enumerator.new do |enum|
|
||||
gathered_lines = [] # collects all lines belonging to a field
|
||||
|
||||
gathered_lines = '' # collects all lines belonging to a field
|
||||
data.each_line do |line|
|
||||
#Rails.logger.debug("___" + line)
|
||||
if line.chomp.empty? # empty line found that seperates paragraphs
|
||||
unless gathered_lines.empty? # any lines gathered so far?
|
||||
if gathered_lines.any? # any lines gathered so far?
|
||||
# Rails.logger.debug('>>>>>>>>>>>>>>>>>>>>>> Gathered lines:')
|
||||
# Rails.logger.debug(gathered_lines.inspect)
|
||||
# Rails.logger.debug('<<<<<<<<<<<<<<<<<<<<<<')
|
||||
enum.yield get_fields(gathered_lines)
|
||||
gathered_lines = ''
|
||||
gathered_lines = []
|
||||
end
|
||||
else
|
||||
gathered_lines << line
|
||||
end
|
||||
end # each_line
|
||||
end
|
||||
|
||||
# Any lines left after the last empty line and the end of the input?
|
||||
unless gathered_lines.empty?
|
||||
enum.yield get_fields(gathered_lines)
|
||||
end
|
||||
end # Enumerator
|
||||
end # def
|
||||
enum.yield get_fields(gathered_lines) if gathered_lines.any?
|
||||
end
|
||||
end
|
||||
|
||||
# Represents the version of a Debian package
|
||||
class Version
|
||||
attr_reader :epoch, :upstream, :revision, :version_string
|
||||
|
||||
|
|
@ -156,10 +168,11 @@ module DebImporter
|
|||
unless /^((?<epoch>\d+)\:)?(?<upstream>.+?)(\-(?<revision>.+))?$/ =~ version_string
|
||||
raise ArgumentError, "Cannot parse version string: #{version_string}"
|
||||
end
|
||||
@epoch = epoch ? epoch : '0'
|
||||
|
||||
@epoch = epoch || '0'
|
||||
@upstream = upstream
|
||||
@revision = revision if revision
|
||||
end # /def
|
||||
end
|
||||
|
||||
def to_s
|
||||
@version_string
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
require 'open-uri' # allows to load URLs using open()
|
||||
require 'deb_importer'
|
||||
|
||||
# The repository format is documented at:
|
||||
|
|
@ -15,7 +14,7 @@ BLACKLIST_NAME_PATTERN=[
|
|||
/-dbg$/,
|
||||
/-common$/,
|
||||
/-l10n($|-)/,
|
||||
/-locale-/,
|
||||
/-locale-/
|
||||
]
|
||||
|
||||
# List of regular expressions. If the package's section matches
|
||||
|
|
@ -23,13 +22,13 @@ BLACKLIST_NAME_PATTERN=[
|
|||
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
|
||||
BLACKLIST_SECTION_PATTERN = [
|
||||
/^debian-installer$/,
|
||||
/\/?translations$/,
|
||||
/\/?debug$/,
|
||||
/\/?kernel$/,
|
||||
/\/?localization$/,
|
||||
/\/?oldlibs$/,
|
||||
/\/?libdevel$/,
|
||||
/\/?cli-mono$/,
|
||||
%r{/?translations$},
|
||||
%r{/?debug$},
|
||||
%r{/?kernel$},
|
||||
%r{/?localization$},
|
||||
%r{/?oldlibs$},
|
||||
%r{/?libdevel$},
|
||||
%r{/?cli-mono$},
|
||||
]
|
||||
|
||||
# Whether to delete a blacklisted package from the database
|
||||
|
|
@ -46,8 +45,8 @@ namespace :debshots do
|
|||
|
||||
repositories = Rails.configuration.package_sources
|
||||
Rails.logger = Logger.new(STDOUT)
|
||||
Rails.logger.level = Logger::INFO
|
||||
#Rails.logger.level = Logger::DEBUG
|
||||
# Rails.logger.level = Logger::INFO
|
||||
Rails.logger.level = Logger::DEBUG
|
||||
|
||||
Rails.logger.info "Importing Debian package information"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue