diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb new file mode 100644 index 0000000..05d3e89 --- /dev/null +++ b/lib/deb_importer.rb @@ -0,0 +1,125 @@ +# Various helper methods to update the database of packages + +module DebImporter + + # This module imports information about packages of a Linux distribution + # that uses the DEB package format like Debian, Ubuntu or Mint. + # + # It starts by loading the Release file of a release to get + # information about available components and architectures. + # + # Next it loads the Packages lists (prefers bz2, falls back to + # .gz or even the uncompressed version). + + class Release + attr_reader :architectures, :components, :description, :codename, :origin, :version, :files + + # Load and parse a Release file of an APT repository + def initialize(dist_url, components) + @dist_url = dist_url + release_url = dist_url + "Release" + Rails.logger.debug "Loading Release file from #{release_url}" + open(release_url) do |release_data| + fields = get_fields(release_data) + @architectures = fields[:Architectures] + @components = fields[:Components] + @description = fields[:Description] + @codename = fields[:Codename] + @origin = fields[:Origin] + @version = fields[:Version] + # Not needed at the moment. We do not check sizes or checksums yet. + #when 'SHA1', 'SHA256', 'MD5Sum' + # next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them + # @files = [] + # value.lines.each do |line| + # path = line.split.last + # @files << path + # end + end # open + end # def initialize + + # Try to load the Packages file for a certain component (e.g. "main") + # and architecture (e.g. "amd64") + def packages(component, architecture) + # create path like "main/binary-amd64/Packages" + packages_path = "#{component}/binary-#{architecture}/Packages" + # Check if gzip or uncompressed files exist + # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) + for suffix in ['.gz', ''] + packages_path_with_suffix = packages_path+suffix + Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" + url = @dist_url + packages_path_with_suffix + begin + Rails.logger.debug "Try opening URL: #{url}" + file = open(url) + if suffix == '.gz' + file = Zlib::GzipReader.new(file) + end + Rails.logger.debug "File containing packages is: #{file}" + #paragraphs = get_paragraphs(file) + #return paragraphs + #byebug + return get_paragraphs(file) + rescue Errno::ENOENT + Rails.logger.debug "URL could not be opened. Skipping." + end + end + return nil + end # def packages + + end # class Release + + private + + # Gather the fields of a Debian control file and return them as a hash + def get_fields(data) + fields = {} + + name=value='' + data.each_line do |line| + case line + when /^(.+): (.+)/ # "Key: Value" + fields[name.to_sym]=value unless value.empty? + name,value=$1,$2 + when /^(.+):\s*/ # "Key:" (start of multi-line entry) + fields[name.to_sym]=value unless value.empty? + name=$1 + value='' + when /^\s+(.+)/ # " Indented multi-line value" + value << $1+"\n" + when /^\s+$/ # Empty line + break + end + end + # Any lines left at the end of the input? + fields[name.to_sym]=value unless value.empty? + + return fields + end + + # Iterator that splits up the input of a debian control file + # by empty lines. For example Debian "Packages" files consist + # of one paragraph for each package listed in it. + def get_paragraphs(data) + Enumerator.new do |enum| + + gathered_lines = '' # collects all lines belonging to a field + data.each_line do |line| + if line.chomp.empty? # empty line found that seperates paragraphs + unless gathered_lines.empty? # any lines gathered so far? + enum.yield get_fields(gathered_lines) + gathered_lines = '' + end + else + gathered_lines << line + end + end # each_line + + # Any lines left after the last empty line and the end of the input? + unless gathered_lines.empty? + enum.yield get_fields(gathered_lines) + end + end # Enumerator + end # def + +end # module