DEB importer can now handle bzip2 files

Gemfile

@@ -93,3 +93,6 @@ gem "paperclip", "~> 4.2"
 
 # Comply with stupid european cookie law
 gem 'cookies_eu'
+
+# Requires: apt-get install libbz2-dev
+gem 'bzip2-ruby', :git => 'https://github.com/chewi/bzip2-ruby.git'

Gemfile.lock

@@ -1,3 +1,9 @@
+GIT
+  remote: https://github.com/chewi/bzip2-ruby.git
+  revision: 92d492c2b40c32a5c039314666655ebc2f52a817
+  specs:
+    bzip2-ruby (0.2.7)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -191,6 +197,7 @@ DEPENDENCIES
   better_errors
   binding_of_caller
   byebug
+  bzip2-ruby!
   coffee-rails (~> 4.1.0)
   cookies_eu
   fancybox2-rails (~> 0.2.8)

lib/deb_importer.rb

@@ -1,5 +1,7 @@
 # Various helper methods to update the database of packages
 
+require 'bzip2'
+
 module DebImporter
 
   # This module imports information about packages of a Linux distribution
@@ -45,15 +47,24 @@ module DebImporter
       packages_path = "/#{component}/binary-#{architecture}/Packages"
       # Check if gzip or uncompressed files exist
       # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014)
-      for suffix in ['.gz', '']
+      for suffix in ['.bz2', '.gz', '']
         packages_path_with_suffix = packages_path+suffix
         Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}"
         url = @dist_url + packages_path_with_suffix
         begin
           Rails.logger.debug "Try opening URL: #{url}"
-          file = open(url)
+          begin
+            file = open(url)
+            puts "URL: #{url}"
+            puts "File: #{file}"
+          rescue OpenURI::HTTPError => e
+            Rails.logger.info "URL #{url} lead to #{e}. skipping."
+            next
+          end
           if suffix == '.gz'
             file = Zlib::GzipReader.new(file)
+          elsif suffix == '.bz2'
+            file = Bzip2::Reader.new(file)
           end
           Rails.logger.debug "File containing packages is: #{file}"
           #paragraphs = get_paragraphs(file)