diff --git a/Gemfile b/Gemfile index 50a9289..ad00aa2 100644 --- a/Gemfile +++ b/Gemfile @@ -93,3 +93,6 @@ gem "paperclip", "~> 4.2" # Comply with stupid european cookie law gem 'cookies_eu' + +# Requires: apt-get install libbz2-dev +gem 'bzip2-ruby', :git => 'https://github.com/chewi/bzip2-ruby.git' diff --git a/Gemfile.lock b/Gemfile.lock index 8c7c9a0..09d4f11 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,9 @@ +GIT + remote: https://github.com/chewi/bzip2-ruby.git + revision: 92d492c2b40c32a5c039314666655ebc2f52a817 + specs: + bzip2-ruby (0.2.7) + GEM remote: https://rubygems.org/ specs: @@ -191,6 +197,7 @@ DEPENDENCIES better_errors binding_of_caller byebug + bzip2-ruby! coffee-rails (~> 4.1.0) cookies_eu fancybox2-rails (~> 0.2.8) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 8934793..3f6607c 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -1,5 +1,7 @@ # Various helper methods to update the database of packages +require 'bzip2' + module DebImporter # This module imports information about packages of a Linux distribution @@ -45,15 +47,24 @@ module DebImporter packages_path = "/#{component}/binary-#{architecture}/Packages" # Check if gzip or uncompressed files exist # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) - for suffix in ['.gz', ''] + for suffix in ['.bz2', '.gz', ''] packages_path_with_suffix = packages_path+suffix Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" url = @dist_url + packages_path_with_suffix begin Rails.logger.debug "Try opening URL: #{url}" - file = open(url) + begin + file = open(url) + puts "URL: #{url}" + puts "File: #{file}" + rescue OpenURI::HTTPError => e + Rails.logger.info "URL #{url} lead to #{e}. skipping." + next + end if suffix == '.gz' file = Zlib::GzipReader.new(file) + elsif suffix == '.bz2' + file = Bzip2::Reader.new(file) end Rails.logger.debug "File containing packages is: #{file}" #paragraphs = get_paragraphs(file)