diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..0780d38 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,14 @@ +# Style/Encoding: +# Enabled: false + +Layout/LineLength: + Max: 99 + +Lint/MixedRegexpCaptureTypes: + Enabled: false + +Metrics/MethodLength: + Max: 30 + +Style/PerlBackrefs: + Enabled: false \ No newline at end of file diff --git a/Gemfile b/Gemfile index 4647833..1772505 100644 --- a/Gemfile +++ b/Gemfile @@ -93,6 +93,8 @@ group :development, :test do gem 'guard-minitest' gem 'minitest-reporters' + gem 'debug' + # Call 'byebug' anywhere in the code to stop execution and get a debugger console gem 'byebug', platform: :mri end @@ -156,3 +158,5 @@ gem 'cancancan' # Gravatars gem 'gravtastic' + +gem 'bzip2-ffi' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index 0b25817..2246d20 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,6 +101,8 @@ GEM msgpack (~> 1.2) builder (3.2.4) byebug (11.1.3) + bzip2-ffi (1.1.1) + ffi (~> 1.0) cancancan (3.5.0) capybara (3.39.2) addressable @@ -118,6 +120,9 @@ GEM content_disposition (1.0.0) crass (1.0.6) date (3.3.4) + debug (1.9.2) + irb (~> 1.10) + reline (>= 0.3.8) debug_inspector (1.2.0) devise (4.9.3) bcrypt (~> 3.0) @@ -485,8 +490,10 @@ DEPENDENCIES binding_of_caller bootsnap byebug + bzip2-ffi cancancan capybara + debug devise fastimage font-awesome-rails @@ -529,4 +536,4 @@ RUBY VERSION ruby 3.2.1p31 BUNDLED WITH - 2.2.33 + 2.5.14 diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 7f26c83..06b355b 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -1,25 +1,30 @@ -# Various helper methods to update the database of packages +# frozen_string_literal: true +require 'open-uri' +require 'bzip2/ffi' +require 'pp' + +# Various helper methods to update the database of packages +# This module imports information about packages of a Linux distribution +# that uses the DEB package format like Debian, Ubuntu or Mint. +# +# It starts by loading the Release file of a release to get +# information about available components and architectures. +# +# Next it loads the Packages lists (prefers bz2, falls back to +# .gz or even the uncompressed version). module DebImporter - # This module imports information about packages of a Linux distribution - # that uses the DEB package format like Debian, Ubuntu or Mint. - # - # It starts by loading the Release file of a release to get - # information about available components and architectures. - # - # Next it loads the Packages lists (prefers bz2, falls back to - # .gz or even the uncompressed version). - + # Defines a Debian release class Release attr_reader :architectures, :components, :description, :codename, :origin, :version, :files # Load and parse a Release file of an APT repository - def initialize(dist_url) + def initialize(dist_url) # rubocop:disable Metrics/MethodLength @dist_url = dist_url - release_url = dist_url + "/Release" + release_url = "#{dist_url}/Release" Rails.logger.debug "Loading Release file from #{release_url}" - open(release_url) do |release_data| + URI.open(release_url) do |release_data| fields = get_fields(release_data) @architectures = fields[:Architectures] @components = fields[:Components] @@ -29,13 +34,13 @@ module DebImporter @version = fields[:Version] # TODO: check sizes and checksums - end # open - end # def initialize + end + end # Get package information from translation (i18n) files. # Returns an enumerator of packages. def i18n(component, language) - url = "#{@dist_url}/#{component}/i18n/Translation-en" + url = "#{@dist_url}/#{component}/i18n/Translation-#{language}" file = find_and_open_compressed_url(url) if file return get_paragraphs(file) @@ -48,34 +53,36 @@ module DebImporter # (e.g. bz2, gz) and fall back to plain text format. def find_and_open_compressed_url(base_url) Rails.logger.debug "Looking for files at URL #{base_url} with different compressions" - for suffix in ['.bz2', '.gz', ''] + ['.bz2', '.gz', ''].each do |suffix| begin - begin - url = "#{base_url}#{suffix}" - Rails.logger.debug "Checking if file at #{url} is available" - file = open(url) - rescue OpenURI::HTTPError => e - Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." - next - end - - # Decompress file depending on its filename suffix - case suffix - when '.bz2' - file = Bzip2::Reader.new(file) - when '.gz' - file = Zlib::GzipReader.new(file) - end - Rails.logger.debug "File containing translations is: #{file}" - - return file - rescue Errno::ENOENT - Rails.logger.debug "URL #{url} could not be opened. Skipping." + url = "#{base_url}#{suffix}" + Rails.logger.debug "Checking if file at #{url} is available" + file = URI.open(url) + rescue OpenURI::HTTPError => e + Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." + next end + + file2 = nil + + # Decompress file depending on its filename suffix + case suffix + when '.bz2' + file2 = Bzip2::FFI::Reader.read(file) + when '.gz' + file2 = Zlib::GzipReader.new(file) + else # plain text + file2 = file + end + Rails.logger.debug "File containing translations is: #{url}" + + # Return an enumerator that iterates over lines of the file + return file2 + rescue Errno::ENOENT + Rails.logger.debug "URL #{url} could not be opened. Skipping." end Rails.logger.error "No file found at #{url} and various compression extensions." - return nil end # Try to load the Packages file for a certain component (e.g. "main") @@ -85,9 +92,9 @@ module DebImporter packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages" Rails.logger.debug "Loading packages from #{packages_path}" file = find_and_open_compressed_url(packages_path) - return get_paragraphs(file) - end # def packages - end # class Release + get_paragraphs(file) + end + end private @@ -95,22 +102,21 @@ module DebImporter def get_fields(data) fields = {} - name = value = "" - data.each_line do |line| + name = value = '' + data.each do |line| case line when /^(\S+?): (.+)/ # "Key: Value" fields[name.to_sym] = value unless value.empty? - name, value = $1, $2 + name = $1 + value = $2 when /^(\S+?):$/ # "Key:" (start of multi-line entry without value in line) fields[name.to_sym] = value unless value.empty? name = $1 value = "" when /^\s(.+)/ # " Indented multi-line value" # Add a newline for multi-line entries ("Key: Value\n Foo\n Bar") - unless value.empty? - value << "\n" - end - value << $1 + value += "\n" unless value.empty? + value += $1 when /^\s+$/ # Empty line break end @@ -118,34 +124,40 @@ module DebImporter # Any lines left at the end of the input? fields[name.to_sym] = value unless value.empty? - return fields + # pp fields + # puts '------------' + fields end # Iterator that splits up the input of a debian control file # by empty lines. For example Debian "Packages" files consist # of one paragraph for each package listed in it. def get_paragraphs(data) + Rails.logger.debug("get_paragraphs – data: #{data.class}") Enumerator.new do |enum| + gathered_lines = [] # collects all lines belonging to a field - gathered_lines = '' # collects all lines belonging to a field data.each_line do |line| + #Rails.logger.debug("___" + line) if line.chomp.empty? # empty line found that seperates paragraphs - unless gathered_lines.empty? # any lines gathered so far? + if gathered_lines.any? # any lines gathered so far? + # Rails.logger.debug('>>>>>>>>>>>>>>>>>>>>>> Gathered lines:') + # Rails.logger.debug(gathered_lines.inspect) + # Rails.logger.debug('<<<<<<<<<<<<<<<<<<<<<<') enum.yield get_fields(gathered_lines) - gathered_lines = '' + gathered_lines = [] end else gathered_lines << line end - end # each_line + end # Any lines left after the last empty line and the end of the input? - unless gathered_lines.empty? - enum.yield get_fields(gathered_lines) - end - end # Enumerator - end # def + enum.yield get_fields(gathered_lines) if gathered_lines.any? + end + end + # Represents the version of a Debian package class Version attr_reader :epoch, :upstream, :revision, :version_string @@ -156,10 +168,11 @@ module DebImporter unless /^((?\d+)\:)?(?.+?)(\-(?.+))?$/ =~ version_string raise ArgumentError, "Cannot parse version string: #{version_string}" end - @epoch = epoch ? epoch : '0' + + @epoch = epoch || '0' @upstream = upstream @revision = revision if revision - end # /def + end def to_s @version_string diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 0438757..9a41fbd 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -1,4 +1,3 @@ -require 'open-uri' # allows to load URLs using open() require 'deb_importer' # The repository format is documented at: @@ -15,7 +14,7 @@ BLACKLIST_NAME_PATTERN=[ /-dbg$/, /-common$/, /-l10n($|-)/, - /-locale-/, + /-locale-/ ] # List of regular expressions. If the package's section matches @@ -23,13 +22,13 @@ BLACKLIST_NAME_PATTERN=[ # /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug BLACKLIST_SECTION_PATTERN = [ /^debian-installer$/, - /\/?translations$/, - /\/?debug$/, - /\/?kernel$/, - /\/?localization$/, - /\/?oldlibs$/, - /\/?libdevel$/, - /\/?cli-mono$/, + %r{/?translations$}, + %r{/?debug$}, + %r{/?kernel$}, + %r{/?localization$}, + %r{/?oldlibs$}, + %r{/?libdevel$}, + %r{/?cli-mono$}, ] # Whether to delete a blacklisted package from the database @@ -46,8 +45,8 @@ namespace :debshots do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::INFO - #Rails.logger.level = Logger::DEBUG + # Rails.logger.level = Logger::INFO + Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing Debian package information" diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index 88a9d35..6c56017 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -3,6 +3,7 @@ $LOAD_PATH << 'lib' require 'open-uri' # allows to load URLs using open() require 'deb_importer' require 'test_helper' +require 'pp' include DebImporter @@ -48,8 +49,8 @@ class PackagesHelperTest < ActionView::TestCase test "should be able to parse local Debian repository test files" do Rails.logger = Logger.new(STDOUT) # Rails.logger.level = Logger::ERROR - Rails.logger.level = Logger::INFO - # Rails.logger.level = Logger::DEBUG + # Rails.logger.level = Logger::INFO + Rails.logger.level = Logger::DEBUG # Load information about test repository from environments/test.rb repositories = Rails.configuration.package_sources @@ -64,6 +65,7 @@ class PackagesHelperTest < ActionView::TestCase packages = release.packages('main', 'amd64') assert_instance_of Enumerator, packages first_package = packages.first + assert_equal 'account-plugin-aim', first_package[:Package] # Packages enumerator must be restarted to start from position 1