bzip2 import fixed

This commit is contained in:
Christoph Haas 2024-07-14 22:42:34 +02:00
parent 1095ae3aa5
commit 48e3a8c67f
6 changed files with 114 additions and 75 deletions

14
.rubocop.yml Normal file
View file

@ -0,0 +1,14 @@
# Style/Encoding:
# Enabled: false
Layout/LineLength:
Max: 99
Lint/MixedRegexpCaptureTypes:
Enabled: false
Metrics/MethodLength:
Max: 30
Style/PerlBackrefs:
Enabled: false

View file

@ -93,6 +93,8 @@ group :development, :test do
gem 'guard-minitest' gem 'guard-minitest'
gem 'minitest-reporters' gem 'minitest-reporters'
gem 'debug'
# Call 'byebug' anywhere in the code to stop execution and get a debugger console # Call 'byebug' anywhere in the code to stop execution and get a debugger console
gem 'byebug', platform: :mri gem 'byebug', platform: :mri
end end
@ -156,3 +158,5 @@ gem 'cancancan'
# Gravatars # Gravatars
gem 'gravtastic' gem 'gravtastic'
gem 'bzip2-ffi'

View file

@ -101,6 +101,8 @@ GEM
msgpack (~> 1.2) msgpack (~> 1.2)
builder (3.2.4) builder (3.2.4)
byebug (11.1.3) byebug (11.1.3)
bzip2-ffi (1.1.1)
ffi (~> 1.0)
cancancan (3.5.0) cancancan (3.5.0)
capybara (3.39.2) capybara (3.39.2)
addressable addressable
@ -118,6 +120,9 @@ GEM
content_disposition (1.0.0) content_disposition (1.0.0)
crass (1.0.6) crass (1.0.6)
date (3.3.4) date (3.3.4)
debug (1.9.2)
irb (~> 1.10)
reline (>= 0.3.8)
debug_inspector (1.2.0) debug_inspector (1.2.0)
devise (4.9.3) devise (4.9.3)
bcrypt (~> 3.0) bcrypt (~> 3.0)
@ -485,8 +490,10 @@ DEPENDENCIES
binding_of_caller binding_of_caller
bootsnap bootsnap
byebug byebug
bzip2-ffi
cancancan cancancan
capybara capybara
debug
devise devise
fastimage fastimage
font-awesome-rails font-awesome-rails
@ -529,4 +536,4 @@ RUBY VERSION
ruby 3.2.1p31 ruby 3.2.1p31
BUNDLED WITH BUNDLED WITH
2.2.33 2.5.14

View file

@ -1,25 +1,30 @@
# Various helper methods to update the database of packages # frozen_string_literal: true
require 'open-uri'
require 'bzip2/ffi'
require 'pp'
# Various helper methods to update the database of packages
# This module imports information about packages of a Linux distribution
# that uses the DEB package format like Debian, Ubuntu or Mint.
#
# It starts by loading the Release file of a release to get
# information about available components and architectures.
#
# Next it loads the Packages lists (prefers bz2, falls back to
# .gz or even the uncompressed version).
module DebImporter module DebImporter
# This module imports information about packages of a Linux distribution # Defines a Debian release
# that uses the DEB package format like Debian, Ubuntu or Mint.
#
# It starts by loading the Release file of a release to get
# information about available components and architectures.
#
# Next it loads the Packages lists (prefers bz2, falls back to
# .gz or even the uncompressed version).
class Release class Release
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
# Load and parse a Release file of an APT repository # Load and parse a Release file of an APT repository
def initialize(dist_url) def initialize(dist_url) # rubocop:disable Metrics/MethodLength
@dist_url = dist_url @dist_url = dist_url
release_url = dist_url + "/Release" release_url = "#{dist_url}/Release"
Rails.logger.debug "Loading Release file from #{release_url}" Rails.logger.debug "Loading Release file from #{release_url}"
open(release_url) do |release_data| URI.open(release_url) do |release_data|
fields = get_fields(release_data) fields = get_fields(release_data)
@architectures = fields[:Architectures] @architectures = fields[:Architectures]
@components = fields[:Components] @components = fields[:Components]
@ -29,13 +34,13 @@ module DebImporter
@version = fields[:Version] @version = fields[:Version]
# TODO: check sizes and checksums # TODO: check sizes and checksums
end # open end
end # def initialize end
# Get package information from translation (i18n) files. # Get package information from translation (i18n) files.
# Returns an enumerator of packages. # Returns an enumerator of packages.
def i18n(component, language) def i18n(component, language)
url = "#{@dist_url}/#{component}/i18n/Translation-en" url = "#{@dist_url}/#{component}/i18n/Translation-#{language}"
file = find_and_open_compressed_url(url) file = find_and_open_compressed_url(url)
if file if file
return get_paragraphs(file) return get_paragraphs(file)
@ -48,34 +53,36 @@ module DebImporter
# (e.g. bz2, gz) and fall back to plain text format. # (e.g. bz2, gz) and fall back to plain text format.
def find_and_open_compressed_url(base_url) def find_and_open_compressed_url(base_url)
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions" Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
for suffix in ['.bz2', '.gz', ''] ['.bz2', '.gz', ''].each do |suffix|
begin begin
begin url = "#{base_url}#{suffix}"
url = "#{base_url}#{suffix}" Rails.logger.debug "Checking if file at #{url} is available"
Rails.logger.debug "Checking if file at #{url} is available" file = URI.open(url)
file = open(url) rescue OpenURI::HTTPError => e
rescue OpenURI::HTTPError => e Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." next
next
end
# Decompress file depending on its filename suffix
case suffix
when '.bz2'
file = Bzip2::Reader.new(file)
when '.gz'
file = Zlib::GzipReader.new(file)
end
Rails.logger.debug "File containing translations is: #{file}"
return file
rescue Errno::ENOENT
Rails.logger.debug "URL #{url} could not be opened. Skipping."
end end
file2 = nil
# Decompress file depending on its filename suffix
case suffix
when '.bz2'
file2 = Bzip2::FFI::Reader.read(file)
when '.gz'
file2 = Zlib::GzipReader.new(file)
else # plain text
file2 = file
end
Rails.logger.debug "File containing translations is: #{url}"
# Return an enumerator that iterates over lines of the file
return file2
rescue Errno::ENOENT
Rails.logger.debug "URL #{url} could not be opened. Skipping."
end end
Rails.logger.error "No file found at #{url} and various compression extensions." Rails.logger.error "No file found at #{url} and various compression extensions."
return nil
end end
# Try to load the Packages file for a certain component (e.g. "main") # Try to load the Packages file for a certain component (e.g. "main")
@ -85,9 +92,9 @@ module DebImporter
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages" packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
Rails.logger.debug "Loading packages from #{packages_path}" Rails.logger.debug "Loading packages from #{packages_path}"
file = find_and_open_compressed_url(packages_path) file = find_and_open_compressed_url(packages_path)
return get_paragraphs(file) get_paragraphs(file)
end # def packages end
end # class Release end
private private
@ -95,22 +102,21 @@ module DebImporter
def get_fields(data) def get_fields(data)
fields = {} fields = {}
name = value = "" name = value = ''
data.each_line do |line| data.each do |line|
case line case line
when /^(\S+?): (.+)/ # "Key: Value" when /^(\S+?): (.+)/ # "Key: Value"
fields[name.to_sym] = value unless value.empty? fields[name.to_sym] = value unless value.empty?
name, value = $1, $2 name = $1
value = $2
when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line) when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line)
fields[name.to_sym] = value unless value.empty? fields[name.to_sym] = value unless value.empty?
name = $1 name = $1
value = "" value = ""
when /^\s(.+)/ # " Indented multi-line value" when /^\s(.+)/ # " Indented multi-line value"
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar") # Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
unless value.empty? value += "\n" unless value.empty?
value << "\n" value += $1
end
value << $1
when /^\s+$/ # Empty line when /^\s+$/ # Empty line
break break
end end
@ -118,34 +124,40 @@ module DebImporter
# Any lines left at the end of the input? # Any lines left at the end of the input?
fields[name.to_sym] = value unless value.empty? fields[name.to_sym] = value unless value.empty?
return fields # pp fields
# puts '------------'
fields
end end
# Iterator that splits up the input of a debian control file # Iterator that splits up the input of a debian control file
# by empty lines. For example Debian "Packages" files consist # by empty lines. For example Debian "Packages" files consist
# of one paragraph for each package listed in it. # of one paragraph for each package listed in it.
def get_paragraphs(data) def get_paragraphs(data)
Rails.logger.debug("get_paragraphs data: #{data.class}")
Enumerator.new do |enum| Enumerator.new do |enum|
gathered_lines = [] # collects all lines belonging to a field
gathered_lines = '' # collects all lines belonging to a field
data.each_line do |line| data.each_line do |line|
#Rails.logger.debug("___" + line)
if line.chomp.empty? # empty line found that seperates paragraphs if line.chomp.empty? # empty line found that seperates paragraphs
unless gathered_lines.empty? # any lines gathered so far? if gathered_lines.any? # any lines gathered so far?
# Rails.logger.debug('>>>>>>>>>>>>>>>>>>>>>> Gathered lines:')
# Rails.logger.debug(gathered_lines.inspect)
# Rails.logger.debug('<<<<<<<<<<<<<<<<<<<<<<')
enum.yield get_fields(gathered_lines) enum.yield get_fields(gathered_lines)
gathered_lines = '' gathered_lines = []
end end
else else
gathered_lines << line gathered_lines << line
end end
end # each_line end
# Any lines left after the last empty line and the end of the input? # Any lines left after the last empty line and the end of the input?
unless gathered_lines.empty? enum.yield get_fields(gathered_lines) if gathered_lines.any?
enum.yield get_fields(gathered_lines) end
end end
end # Enumerator
end # def
# Represents the version of a Debian package
class Version class Version
attr_reader :epoch, :upstream, :revision, :version_string attr_reader :epoch, :upstream, :revision, :version_string
@ -156,10 +168,11 @@ module DebImporter
unless /^((?<epoch>\d+)\:)?(?<upstream>.+?)(\-(?<revision>.+))?$/ =~ version_string unless /^((?<epoch>\d+)\:)?(?<upstream>.+?)(\-(?<revision>.+))?$/ =~ version_string
raise ArgumentError, "Cannot parse version string: #{version_string}" raise ArgumentError, "Cannot parse version string: #{version_string}"
end end
@epoch = epoch ? epoch : '0'
@epoch = epoch || '0'
@upstream = upstream @upstream = upstream
@revision = revision if revision @revision = revision if revision
end # /def end
def to_s def to_s
@version_string @version_string

View file

@ -1,4 +1,3 @@
require 'open-uri' # allows to load URLs using open()
require 'deb_importer' require 'deb_importer'
# The repository format is documented at: # The repository format is documented at:
@ -15,7 +14,7 @@ BLACKLIST_NAME_PATTERN=[
/-dbg$/, /-dbg$/,
/-common$/, /-common$/,
/-l10n($|-)/, /-l10n($|-)/,
/-locale-/, /-locale-/
] ]
# List of regular expressions. If the package's section matches # List of regular expressions. If the package's section matches
@ -23,13 +22,13 @@ BLACKLIST_NAME_PATTERN=[
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug # /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
BLACKLIST_SECTION_PATTERN = [ BLACKLIST_SECTION_PATTERN = [
/^debian-installer$/, /^debian-installer$/,
/\/?translations$/, %r{/?translations$},
/\/?debug$/, %r{/?debug$},
/\/?kernel$/, %r{/?kernel$},
/\/?localization$/, %r{/?localization$},
/\/?oldlibs$/, %r{/?oldlibs$},
/\/?libdevel$/, %r{/?libdevel$},
/\/?cli-mono$/, %r{/?cli-mono$},
] ]
# Whether to delete a blacklisted package from the database # Whether to delete a blacklisted package from the database
@ -46,8 +45,8 @@ namespace :debshots do
repositories = Rails.configuration.package_sources repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT) Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::INFO
#Rails.logger.level = Logger::DEBUG Rails.logger.level = Logger::DEBUG
Rails.logger.info "Importing Debian package information" Rails.logger.info "Importing Debian package information"

View file

@ -3,6 +3,7 @@ $LOAD_PATH << 'lib'
require 'open-uri' # allows to load URLs using open() require 'open-uri' # allows to load URLs using open()
require 'deb_importer' require 'deb_importer'
require 'test_helper' require 'test_helper'
require 'pp'
include DebImporter include DebImporter
@ -48,8 +49,8 @@ class PackagesHelperTest < ActionView::TestCase
test "should be able to parse local Debian repository test files" do test "should be able to parse local Debian repository test files" do
Rails.logger = Logger.new(STDOUT) Rails.logger = Logger.new(STDOUT)
# Rails.logger.level = Logger::ERROR # Rails.logger.level = Logger::ERROR
Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::INFO
# Rails.logger.level = Logger::DEBUG Rails.logger.level = Logger::DEBUG
# Load information about test repository from environments/test.rb # Load information about test repository from environments/test.rb
repositories = Rails.configuration.package_sources repositories = Rails.configuration.package_sources
@ -64,6 +65,7 @@ class PackagesHelperTest < ActionView::TestCase
packages = release.packages('main', 'amd64') packages = release.packages('main', 'amd64')
assert_instance_of Enumerator, packages assert_instance_of Enumerator, packages
first_package = packages.first first_package = packages.first
assert_equal 'account-plugin-aim', first_package[:Package] assert_equal 'account-plugin-aim', first_package[:Package]
# Packages enumerator must be restarted to start from position 1 # Packages enumerator must be restarted to start from position 1