# Various helper methods to update the database of packages require 'bzip2' module DebImporter # This module imports information about packages of a Linux distribution # that uses the DEB package format like Debian, Ubuntu or Mint. # # It starts by loading the Release file of a release to get # information about available components and architectures. # # Next it loads the Packages lists (prefers bz2, falls back to # .gz or even the uncompressed version). class Release attr_reader :architectures, :components, :description, :codename, :origin, :version, :files # Load and parse a Release file of an APT repository def initialize(dist_url, components) @dist_url = dist_url release_url = dist_url + "/Release" Rails.logger.debug "Loading Release file from #{release_url}" open(release_url) do |release_data| fields = get_fields(release_data) @architectures = fields[:Architectures] @components = fields[:Components] @description = fields[:Description] @codename = fields[:Codename] @origin = fields[:Origin] @version = fields[:Version] # Not needed at the moment. We do not check sizes or checksums yet. #when 'SHA1', 'SHA256', 'MD5Sum' # next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them # @files = [] # value.lines.each do |line| # path = line.split.last # @files << path # end end # open end # def initialize # TODO: Avoid code duplication (bz2, gz, ...) def i18n(component, language) for suffix in ['.bz2', '.gz', ''] url = "#{@dist_url}/#{component}/i18n/Translation-en#{suffix}" Rails.logger.debug "Looking for translations file: #{url}" begin Rails.logger.debug "Try opening URL: #{url}" begin file = open(url) rescue OpenURI::HTTPError => e Rails.logger.info "URL #{url} lead to #{e}. skipping." next end if suffix == '.gz' file = Zlib::GzipReader.new(file) elsif suffix == '.bz2' file = Bzip2::Reader.new(file) end Rails.logger.debug "File containing translations is: #{file}" return get_paragraphs(file) rescue Errno::ENOENT Rails.logger.debug "URL could not be opened. Skipping." end end end # Try to load the Packages file for a certain component (e.g. "main") # and architecture (e.g. "amd64") def packages(component, architecture) # create path like "main/binary-amd64/Packages" packages_path = "/#{component}/binary-#{architecture}/Packages" # Check if gzip or uncompressed files exist # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) for suffix in ['.bz2', '.gz', ''] packages_path_with_suffix = packages_path+suffix Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" url = @dist_url + packages_path_with_suffix begin Rails.logger.debug "Try opening URL: #{url}" begin file = open(url) rescue OpenURI::HTTPError => e Rails.logger.info "URL #{url} lead to #{e}. skipping." next end if suffix == '.gz' file = Zlib::GzipReader.new(file) elsif suffix == '.bz2' file = Bzip2::Reader.new(file) end Rails.logger.debug "File containing packages is: #{file}" #paragraphs = get_paragraphs(file) #return paragraphs #byebug return get_paragraphs(file) rescue Errno::ENOENT Rails.logger.debug "URL could not be opened. Skipping." end end return nil end # def packages end # class Release private # Gather the fields of a Debian control file and return them as a hash def get_fields(data) fields = {} name=value='' data.each_line do |line| case line when /^(.+): (.+)/ # "Key: Value" fields[name.to_sym]=value unless value.empty? name,value=$1,$2 when /^(.+):\s*/ # "Key:" (start of multi-line entry) fields[name.to_sym]=value unless value.empty? name=$1 value='' when /^\s+(.+)/ # " Indented multi-line value" value << $1+"\n" when /^\s+$/ # Empty line break end end # Any lines left at the end of the input? fields[name.to_sym]=value unless value.empty? return fields end # Iterator that splits up the input of a debian control file # by empty lines. For example Debian "Packages" files consist # of one paragraph for each package listed in it. def get_paragraphs(data) Enumerator.new do |enum| gathered_lines = '' # collects all lines belonging to a field data.each_line do |line| if line.chomp.empty? # empty line found that seperates paragraphs unless gathered_lines.empty? # any lines gathered so far? enum.yield get_fields(gathered_lines) gathered_lines = '' end else gathered_lines << line end end # each_line # Any lines left after the last empty line and the end of the input? unless gathered_lines.empty? enum.yield get_fields(gathered_lines) end end # Enumerator end # def end # module