Script tidied up

lib/tasks/import_debian.rake

@@ -1,20 +1,51 @@
 require 'open-uri'  # allows to load URLs using open()
 require 'deb_importer'
 
+# The repository format is documented at:
+# https://wiki.debian.org/RepositoryFormat
+
 include DebImporter
 
+# List of regular expressions. If the package name matches any
+# of these then the package will not be imported.
 BLACKLIST_NAME_PATTERN=[
-  '-data$',
+  /-data$/,
-  '-dev$',
+  /-dev$/,
-  '-doc$',
+  /-doc$/,
+  /-dbg$/,
+  /-common$/,
+  /-l10n-/,
+  /-locale-/,
 ]
 
+# List of regular expressions. If the package's section matches
+# any of these then the package will not be imported.
+# /?...$ is used because Ubuntu adds their own section - e.g. multiverse/debug
+BLACKLIST_SECTION_PATTERN = [
+  /^debian-installer$/,
+  /\/?text$/,
+  /\/?translations$/,
+  /\/?debug$/,
+  /\/?kernel$/,
+  /\/?libs$/,
+  /\/?localization$/,
+  /\/?oldlibs$/,
+  /\/?otherosfs$/,
+  /\/?doc$/,
+  /\/?libdevel$/,
+  /\/?cli-mono$/,
+]
+
+# Whether to delete a packag
+REMOVE_BLACKLISTED_PACKAGE = true
+
 desc "Import package information from Debian-style Packages files"
 
 task :import_debian => :environment do
   repositories = Rails.configuration.package_sources
   Rails.logger = Logger.new(STDOUT)
-  Rails.logger.level = Logger::DEBUG
+  Rails.logger.level = Logger::INFO
+  #Rails.logger.level = Logger::DEBUG
 
   puts "Importing Debian package information"
 
@@ -41,9 +72,22 @@ task :import_debian => :environment do
         Rails.logger.debug "Got: #{packages}"
         packages.each do |package|
           Rails.logger.info "> Package: #{package[:Package]}"
-          #Rails.logger.debug "Fetching package informaton from our database"
+          #Rails.logger.debug "Fetching package informaton from the database"
-          #db_package = Package[name: package[:Package]]
+
           db_package = Package.find_by name: package[:Package]
+
+          if package_blacklisted? package
+            # Should the package get removed from the database?
+            if REMOVE_BLACKLISTED_PACKAGE
+              if db_package
+                Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
+                db_package.destroy
+              end
+            end
+
+            next
+          end # if blacklisted
+
           unless db_package
             Rails.logger.debug "No such package in our database."
             next
@@ -51,22 +95,19 @@ task :import_debian => :environment do
 
           Rails.logger.debug "Found in database: #{db_package.id}"
           update_data(package, db_package)
-        end
+        end # package.each
 
-      end
+      end # architectures.each
-    end
+    end # components.each
-  end
 
-end
+    Rails.logger.info "Done parsing #{repository[:url]} repository."
+    Rails.logger.info "--------------------------------------------"
+  end # repositories.each
+
+end # task
 
 # Update the information about a package in the database
 def update_data(package, db_package)
-  # Check update criteria (blacklist, whitelist)
-  #for name, name in expression
-    #code
-  #end
-
-
   Rails.logger.debug "New information: #{package.to_yaml}"
   Rails.logger.debug "Database information: #{db_package.to_yaml}"
 
@@ -74,7 +115,6 @@ def update_data(package, db_package)
   # Only the actual upstream software version (e.g. 1.2) but not the package
   # revision (e.g. 4:1.2-ubuntu0)
   package[:Version] =~ /(?:\d+\:)?(.+?)(~|\+|\-|$)/
-  #new_version = package[:Version].split('-')[0]
   new_version = $1
   if not new_version
     Rails.logger.fatal "Could not parse version string: #{package[:Version]}"
@@ -87,7 +127,32 @@ def update_data(package, db_package)
 
   db_package.save
 
-  # TODO: Deal with different distributions differently
-
   # TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
 end
+
+# Check if a package is blacklisted and should not be imported.
+# This prevents importing boring software that likely does not get a useful screenshot.
+# Argument: Hash containing package information from a Packages file
+def package_blacklisted?(package)
+  Rails.logger.debug "Blacklist check of package #{package[:Package]}"
+
+  # TODO: add a flag that allows to manually blacklist a package
+
+  BLACKLIST_SECTION_PATTERN.each do |pattern|
+    Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}"
+    if package[:Section] =~ pattern
+      Rails.logger.info "  > Blacklisted by section (matched pattern '#{pattern.inspect}')"
+      return true
+    end
+  end
+
+  BLACKLIST_NAME_PATTERN.each do |pattern|
+    Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}"
+    if package[:Package] =~ pattern
+      Rails.logger.info "  > Blacklisted by name (matched pattern '#{pattern.inspect}')"
+      return true
+    end
+  end
+
+  false
+end