bzip2 import fixed
This commit is contained in:
parent
1095ae3aa5
commit
48e3a8c67f
6 changed files with 114 additions and 75 deletions
14
.rubocop.yml
Normal file
14
.rubocop.yml
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
# Style/Encoding:
|
||||||
|
# Enabled: false
|
||||||
|
|
||||||
|
Layout/LineLength:
|
||||||
|
Max: 99
|
||||||
|
|
||||||
|
Lint/MixedRegexpCaptureTypes:
|
||||||
|
Enabled: false
|
||||||
|
|
||||||
|
Metrics/MethodLength:
|
||||||
|
Max: 30
|
||||||
|
|
||||||
|
Style/PerlBackrefs:
|
||||||
|
Enabled: false
|
||||||
4
Gemfile
4
Gemfile
|
|
@ -93,6 +93,8 @@ group :development, :test do
|
||||||
gem 'guard-minitest'
|
gem 'guard-minitest'
|
||||||
gem 'minitest-reporters'
|
gem 'minitest-reporters'
|
||||||
|
|
||||||
|
gem 'debug'
|
||||||
|
|
||||||
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
|
# Call 'byebug' anywhere in the code to stop execution and get a debugger console
|
||||||
gem 'byebug', platform: :mri
|
gem 'byebug', platform: :mri
|
||||||
end
|
end
|
||||||
|
|
@ -156,3 +158,5 @@ gem 'cancancan'
|
||||||
|
|
||||||
# Gravatars
|
# Gravatars
|
||||||
gem 'gravtastic'
|
gem 'gravtastic'
|
||||||
|
|
||||||
|
gem 'bzip2-ffi'
|
||||||
|
|
@ -101,6 +101,8 @@ GEM
|
||||||
msgpack (~> 1.2)
|
msgpack (~> 1.2)
|
||||||
builder (3.2.4)
|
builder (3.2.4)
|
||||||
byebug (11.1.3)
|
byebug (11.1.3)
|
||||||
|
bzip2-ffi (1.1.1)
|
||||||
|
ffi (~> 1.0)
|
||||||
cancancan (3.5.0)
|
cancancan (3.5.0)
|
||||||
capybara (3.39.2)
|
capybara (3.39.2)
|
||||||
addressable
|
addressable
|
||||||
|
|
@ -118,6 +120,9 @@ GEM
|
||||||
content_disposition (1.0.0)
|
content_disposition (1.0.0)
|
||||||
crass (1.0.6)
|
crass (1.0.6)
|
||||||
date (3.3.4)
|
date (3.3.4)
|
||||||
|
debug (1.9.2)
|
||||||
|
irb (~> 1.10)
|
||||||
|
reline (>= 0.3.8)
|
||||||
debug_inspector (1.2.0)
|
debug_inspector (1.2.0)
|
||||||
devise (4.9.3)
|
devise (4.9.3)
|
||||||
bcrypt (~> 3.0)
|
bcrypt (~> 3.0)
|
||||||
|
|
@ -485,8 +490,10 @@ DEPENDENCIES
|
||||||
binding_of_caller
|
binding_of_caller
|
||||||
bootsnap
|
bootsnap
|
||||||
byebug
|
byebug
|
||||||
|
bzip2-ffi
|
||||||
cancancan
|
cancancan
|
||||||
capybara
|
capybara
|
||||||
|
debug
|
||||||
devise
|
devise
|
||||||
fastimage
|
fastimage
|
||||||
font-awesome-rails
|
font-awesome-rails
|
||||||
|
|
@ -529,4 +536,4 @@ RUBY VERSION
|
||||||
ruby 3.2.1p31
|
ruby 3.2.1p31
|
||||||
|
|
||||||
BUNDLED WITH
|
BUNDLED WITH
|
||||||
2.2.33
|
2.5.14
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,30 @@
|
||||||
# Various helper methods to update the database of packages
|
# frozen_string_literal: true
|
||||||
|
|
||||||
|
require 'open-uri'
|
||||||
|
require 'bzip2/ffi'
|
||||||
|
require 'pp'
|
||||||
|
|
||||||
|
# Various helper methods to update the database of packages
|
||||||
|
# This module imports information about packages of a Linux distribution
|
||||||
|
# that uses the DEB package format like Debian, Ubuntu or Mint.
|
||||||
|
#
|
||||||
|
# It starts by loading the Release file of a release to get
|
||||||
|
# information about available components and architectures.
|
||||||
|
#
|
||||||
|
# Next it loads the Packages lists (prefers bz2, falls back to
|
||||||
|
# .gz or even the uncompressed version).
|
||||||
module DebImporter
|
module DebImporter
|
||||||
|
|
||||||
# This module imports information about packages of a Linux distribution
|
# Defines a Debian release
|
||||||
# that uses the DEB package format like Debian, Ubuntu or Mint.
|
|
||||||
#
|
|
||||||
# It starts by loading the Release file of a release to get
|
|
||||||
# information about available components and architectures.
|
|
||||||
#
|
|
||||||
# Next it loads the Packages lists (prefers bz2, falls back to
|
|
||||||
# .gz or even the uncompressed version).
|
|
||||||
|
|
||||||
class Release
|
class Release
|
||||||
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
|
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
|
||||||
|
|
||||||
# Load and parse a Release file of an APT repository
|
# Load and parse a Release file of an APT repository
|
||||||
def initialize(dist_url)
|
def initialize(dist_url) # rubocop:disable Metrics/MethodLength
|
||||||
@dist_url = dist_url
|
@dist_url = dist_url
|
||||||
release_url = dist_url + "/Release"
|
release_url = "#{dist_url}/Release"
|
||||||
Rails.logger.debug "Loading Release file from #{release_url}"
|
Rails.logger.debug "Loading Release file from #{release_url}"
|
||||||
open(release_url) do |release_data|
|
URI.open(release_url) do |release_data|
|
||||||
fields = get_fields(release_data)
|
fields = get_fields(release_data)
|
||||||
@architectures = fields[:Architectures]
|
@architectures = fields[:Architectures]
|
||||||
@components = fields[:Components]
|
@components = fields[:Components]
|
||||||
|
|
@ -29,13 +34,13 @@ module DebImporter
|
||||||
@version = fields[:Version]
|
@version = fields[:Version]
|
||||||
|
|
||||||
# TODO: check sizes and checksums
|
# TODO: check sizes and checksums
|
||||||
end # open
|
end
|
||||||
end # def initialize
|
end
|
||||||
|
|
||||||
# Get package information from translation (i18n) files.
|
# Get package information from translation (i18n) files.
|
||||||
# Returns an enumerator of packages.
|
# Returns an enumerator of packages.
|
||||||
def i18n(component, language)
|
def i18n(component, language)
|
||||||
url = "#{@dist_url}/#{component}/i18n/Translation-en"
|
url = "#{@dist_url}/#{component}/i18n/Translation-#{language}"
|
||||||
file = find_and_open_compressed_url(url)
|
file = find_and_open_compressed_url(url)
|
||||||
if file
|
if file
|
||||||
return get_paragraphs(file)
|
return get_paragraphs(file)
|
||||||
|
|
@ -48,34 +53,36 @@ module DebImporter
|
||||||
# (e.g. bz2, gz) and fall back to plain text format.
|
# (e.g. bz2, gz) and fall back to plain text format.
|
||||||
def find_and_open_compressed_url(base_url)
|
def find_and_open_compressed_url(base_url)
|
||||||
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
|
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
|
||||||
for suffix in ['.bz2', '.gz', '']
|
['.bz2', '.gz', ''].each do |suffix|
|
||||||
begin
|
begin
|
||||||
begin
|
url = "#{base_url}#{suffix}"
|
||||||
url = "#{base_url}#{suffix}"
|
Rails.logger.debug "Checking if file at #{url} is available"
|
||||||
Rails.logger.debug "Checking if file at #{url} is available"
|
file = URI.open(url)
|
||||||
file = open(url)
|
rescue OpenURI::HTTPError => e
|
||||||
rescue OpenURI::HTTPError => e
|
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
|
||||||
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
|
next
|
||||||
next
|
|
||||||
end
|
|
||||||
|
|
||||||
# Decompress file depending on its filename suffix
|
|
||||||
case suffix
|
|
||||||
when '.bz2'
|
|
||||||
file = Bzip2::Reader.new(file)
|
|
||||||
when '.gz'
|
|
||||||
file = Zlib::GzipReader.new(file)
|
|
||||||
end
|
|
||||||
Rails.logger.debug "File containing translations is: #{file}"
|
|
||||||
|
|
||||||
return file
|
|
||||||
rescue Errno::ENOENT
|
|
||||||
Rails.logger.debug "URL #{url} could not be opened. Skipping."
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
file2 = nil
|
||||||
|
|
||||||
|
# Decompress file depending on its filename suffix
|
||||||
|
case suffix
|
||||||
|
when '.bz2'
|
||||||
|
file2 = Bzip2::FFI::Reader.read(file)
|
||||||
|
when '.gz'
|
||||||
|
file2 = Zlib::GzipReader.new(file)
|
||||||
|
else # plain text
|
||||||
|
file2 = file
|
||||||
|
end
|
||||||
|
Rails.logger.debug "File containing translations is: #{url}"
|
||||||
|
|
||||||
|
# Return an enumerator that iterates over lines of the file
|
||||||
|
return file2
|
||||||
|
rescue Errno::ENOENT
|
||||||
|
Rails.logger.debug "URL #{url} could not be opened. Skipping."
|
||||||
end
|
end
|
||||||
|
|
||||||
Rails.logger.error "No file found at #{url} and various compression extensions."
|
Rails.logger.error "No file found at #{url} and various compression extensions."
|
||||||
return nil
|
|
||||||
end
|
end
|
||||||
|
|
||||||
# Try to load the Packages file for a certain component (e.g. "main")
|
# Try to load the Packages file for a certain component (e.g. "main")
|
||||||
|
|
@ -85,9 +92,9 @@ module DebImporter
|
||||||
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
|
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
|
||||||
Rails.logger.debug "Loading packages from #{packages_path}"
|
Rails.logger.debug "Loading packages from #{packages_path}"
|
||||||
file = find_and_open_compressed_url(packages_path)
|
file = find_and_open_compressed_url(packages_path)
|
||||||
return get_paragraphs(file)
|
get_paragraphs(file)
|
||||||
end # def packages
|
end
|
||||||
end # class Release
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
|
|
@ -95,22 +102,21 @@ module DebImporter
|
||||||
def get_fields(data)
|
def get_fields(data)
|
||||||
fields = {}
|
fields = {}
|
||||||
|
|
||||||
name = value = ""
|
name = value = ''
|
||||||
data.each_line do |line|
|
data.each do |line|
|
||||||
case line
|
case line
|
||||||
when /^(\S+?): (.+)/ # "Key: Value"
|
when /^(\S+?): (.+)/ # "Key: Value"
|
||||||
fields[name.to_sym] = value unless value.empty?
|
fields[name.to_sym] = value unless value.empty?
|
||||||
name, value = $1, $2
|
name = $1
|
||||||
|
value = $2
|
||||||
when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line)
|
when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line)
|
||||||
fields[name.to_sym] = value unless value.empty?
|
fields[name.to_sym] = value unless value.empty?
|
||||||
name = $1
|
name = $1
|
||||||
value = ""
|
value = ""
|
||||||
when /^\s(.+)/ # " Indented multi-line value"
|
when /^\s(.+)/ # " Indented multi-line value"
|
||||||
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
|
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
|
||||||
unless value.empty?
|
value += "\n" unless value.empty?
|
||||||
value << "\n"
|
value += $1
|
||||||
end
|
|
||||||
value << $1
|
|
||||||
when /^\s+$/ # Empty line
|
when /^\s+$/ # Empty line
|
||||||
break
|
break
|
||||||
end
|
end
|
||||||
|
|
@ -118,34 +124,40 @@ module DebImporter
|
||||||
# Any lines left at the end of the input?
|
# Any lines left at the end of the input?
|
||||||
fields[name.to_sym] = value unless value.empty?
|
fields[name.to_sym] = value unless value.empty?
|
||||||
|
|
||||||
return fields
|
# pp fields
|
||||||
|
# puts '------------'
|
||||||
|
fields
|
||||||
end
|
end
|
||||||
|
|
||||||
# Iterator that splits up the input of a debian control file
|
# Iterator that splits up the input of a debian control file
|
||||||
# by empty lines. For example Debian "Packages" files consist
|
# by empty lines. For example Debian "Packages" files consist
|
||||||
# of one paragraph for each package listed in it.
|
# of one paragraph for each package listed in it.
|
||||||
def get_paragraphs(data)
|
def get_paragraphs(data)
|
||||||
|
Rails.logger.debug("get_paragraphs – data: #{data.class}")
|
||||||
Enumerator.new do |enum|
|
Enumerator.new do |enum|
|
||||||
|
gathered_lines = [] # collects all lines belonging to a field
|
||||||
|
|
||||||
gathered_lines = '' # collects all lines belonging to a field
|
|
||||||
data.each_line do |line|
|
data.each_line do |line|
|
||||||
|
#Rails.logger.debug("___" + line)
|
||||||
if line.chomp.empty? # empty line found that seperates paragraphs
|
if line.chomp.empty? # empty line found that seperates paragraphs
|
||||||
unless gathered_lines.empty? # any lines gathered so far?
|
if gathered_lines.any? # any lines gathered so far?
|
||||||
|
# Rails.logger.debug('>>>>>>>>>>>>>>>>>>>>>> Gathered lines:')
|
||||||
|
# Rails.logger.debug(gathered_lines.inspect)
|
||||||
|
# Rails.logger.debug('<<<<<<<<<<<<<<<<<<<<<<')
|
||||||
enum.yield get_fields(gathered_lines)
|
enum.yield get_fields(gathered_lines)
|
||||||
gathered_lines = ''
|
gathered_lines = []
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
gathered_lines << line
|
gathered_lines << line
|
||||||
end
|
end
|
||||||
end # each_line
|
end
|
||||||
|
|
||||||
# Any lines left after the last empty line and the end of the input?
|
# Any lines left after the last empty line and the end of the input?
|
||||||
unless gathered_lines.empty?
|
enum.yield get_fields(gathered_lines) if gathered_lines.any?
|
||||||
enum.yield get_fields(gathered_lines)
|
end
|
||||||
end
|
end
|
||||||
end # Enumerator
|
|
||||||
end # def
|
|
||||||
|
|
||||||
|
# Represents the version of a Debian package
|
||||||
class Version
|
class Version
|
||||||
attr_reader :epoch, :upstream, :revision, :version_string
|
attr_reader :epoch, :upstream, :revision, :version_string
|
||||||
|
|
||||||
|
|
@ -156,10 +168,11 @@ module DebImporter
|
||||||
unless /^((?<epoch>\d+)\:)?(?<upstream>.+?)(\-(?<revision>.+))?$/ =~ version_string
|
unless /^((?<epoch>\d+)\:)?(?<upstream>.+?)(\-(?<revision>.+))?$/ =~ version_string
|
||||||
raise ArgumentError, "Cannot parse version string: #{version_string}"
|
raise ArgumentError, "Cannot parse version string: #{version_string}"
|
||||||
end
|
end
|
||||||
@epoch = epoch ? epoch : '0'
|
|
||||||
|
@epoch = epoch || '0'
|
||||||
@upstream = upstream
|
@upstream = upstream
|
||||||
@revision = revision if revision
|
@revision = revision if revision
|
||||||
end # /def
|
end
|
||||||
|
|
||||||
def to_s
|
def to_s
|
||||||
@version_string
|
@version_string
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,3 @@
|
||||||
require 'open-uri' # allows to load URLs using open()
|
|
||||||
require 'deb_importer'
|
require 'deb_importer'
|
||||||
|
|
||||||
# The repository format is documented at:
|
# The repository format is documented at:
|
||||||
|
|
@ -15,7 +14,7 @@ BLACKLIST_NAME_PATTERN=[
|
||||||
/-dbg$/,
|
/-dbg$/,
|
||||||
/-common$/,
|
/-common$/,
|
||||||
/-l10n($|-)/,
|
/-l10n($|-)/,
|
||||||
/-locale-/,
|
/-locale-/
|
||||||
]
|
]
|
||||||
|
|
||||||
# List of regular expressions. If the package's section matches
|
# List of regular expressions. If the package's section matches
|
||||||
|
|
@ -23,13 +22,13 @@ BLACKLIST_NAME_PATTERN=[
|
||||||
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
|
# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
|
||||||
BLACKLIST_SECTION_PATTERN = [
|
BLACKLIST_SECTION_PATTERN = [
|
||||||
/^debian-installer$/,
|
/^debian-installer$/,
|
||||||
/\/?translations$/,
|
%r{/?translations$},
|
||||||
/\/?debug$/,
|
%r{/?debug$},
|
||||||
/\/?kernel$/,
|
%r{/?kernel$},
|
||||||
/\/?localization$/,
|
%r{/?localization$},
|
||||||
/\/?oldlibs$/,
|
%r{/?oldlibs$},
|
||||||
/\/?libdevel$/,
|
%r{/?libdevel$},
|
||||||
/\/?cli-mono$/,
|
%r{/?cli-mono$},
|
||||||
]
|
]
|
||||||
|
|
||||||
# Whether to delete a blacklisted package from the database
|
# Whether to delete a blacklisted package from the database
|
||||||
|
|
@ -46,8 +45,8 @@ namespace :debshots do
|
||||||
|
|
||||||
repositories = Rails.configuration.package_sources
|
repositories = Rails.configuration.package_sources
|
||||||
Rails.logger = Logger.new(STDOUT)
|
Rails.logger = Logger.new(STDOUT)
|
||||||
Rails.logger.level = Logger::INFO
|
# Rails.logger.level = Logger::INFO
|
||||||
#Rails.logger.level = Logger::DEBUG
|
Rails.logger.level = Logger::DEBUG
|
||||||
|
|
||||||
Rails.logger.info "Importing Debian package information"
|
Rails.logger.info "Importing Debian package information"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ $LOAD_PATH << 'lib'
|
||||||
require 'open-uri' # allows to load URLs using open()
|
require 'open-uri' # allows to load URLs using open()
|
||||||
require 'deb_importer'
|
require 'deb_importer'
|
||||||
require 'test_helper'
|
require 'test_helper'
|
||||||
|
require 'pp'
|
||||||
|
|
||||||
include DebImporter
|
include DebImporter
|
||||||
|
|
||||||
|
|
@ -48,8 +49,8 @@ class PackagesHelperTest < ActionView::TestCase
|
||||||
test "should be able to parse local Debian repository test files" do
|
test "should be able to parse local Debian repository test files" do
|
||||||
Rails.logger = Logger.new(STDOUT)
|
Rails.logger = Logger.new(STDOUT)
|
||||||
# Rails.logger.level = Logger::ERROR
|
# Rails.logger.level = Logger::ERROR
|
||||||
Rails.logger.level = Logger::INFO
|
# Rails.logger.level = Logger::INFO
|
||||||
# Rails.logger.level = Logger::DEBUG
|
Rails.logger.level = Logger::DEBUG
|
||||||
|
|
||||||
# Load information about test repository from environments/test.rb
|
# Load information about test repository from environments/test.rb
|
||||||
repositories = Rails.configuration.package_sources
|
repositories = Rails.configuration.package_sources
|
||||||
|
|
@ -64,6 +65,7 @@ class PackagesHelperTest < ActionView::TestCase
|
||||||
packages = release.packages('main', 'amd64')
|
packages = release.packages('main', 'amd64')
|
||||||
assert_instance_of Enumerator, packages
|
assert_instance_of Enumerator, packages
|
||||||
first_package = packages.first
|
first_package = packages.first
|
||||||
|
|
||||||
assert_equal 'account-plugin-aim', first_package[:Package]
|
assert_equal 'account-plugin-aim', first_package[:Package]
|
||||||
|
|
||||||
# Packages enumerator must be restarted to start from position 1
|
# Packages enumerator must be restarted to start from position 1
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue