From 3707c1c33c43533efabbc1304d97cfc4d6f2c707 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 20 Apr 2015 19:49:12 +0200 Subject: [PATCH 01/43] Added blurred shadow around images --- app/assets/stylesheets/my.scss | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/assets/stylesheets/my.scss b/app/assets/stylesheets/my.scss index 58f8395..0ae1f2c 100644 --- a/app/assets/stylesheets/my.scss +++ b/app/assets/stylesheets/my.scss @@ -148,6 +148,11 @@ html, body { { width: 100%; } + + img + { + box-shadow: 3px 3px 10px 0px #888888; + } } .row .gridview From 20f384c36efde7ceb88cffefff9730d20edcbec9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 20 Apr 2015 20:02:04 +0200 Subject: [PATCH 02/43] Image caption shown in italic --- app/assets/stylesheets/my.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/stylesheets/my.scss b/app/assets/stylesheets/my.scss index 0ae1f2c..81e0468 100644 --- a/app/assets/stylesheets/my.scss +++ b/app/assets/stylesheets/my.scss @@ -232,6 +232,7 @@ a.black .imgcaption { text-align: center; + font-variant: italic; } .bigpanel From 9b3f05bad9c44567f4064b276b1f9e7a949cee74 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 20 Apr 2015 20:02:42 +0200 Subject: [PATCH 03/43] Removed image on the front page. Because there is no useful information like review ratings or popcon scores that would help here. Might be re-enabled later when that information is available. --- app/views/welcome/home.slim | 68 ++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/app/views/welcome/home.slim b/app/views/welcome/home.slim index a08f186..5345053 100644 --- a/app/views/welcome/home.slim +++ b/app/views/welcome/home.slim @@ -25,37 +25,37 @@ div.row hr -div.row - div.large-4.medium-12.small-12.columns.text-center - h2 Newest upload - // TODO: load actual newest upload and cache it - // TODO: link to the package page - a.black href=package_path(@newest_uploaded_package.name) - img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false) - p - strong - =@newest_uploaded_package.name - ' : - ' - =@newest_uploaded_package.description - div.large-4.medium-12.small-12.columns.text-center - h2 Most popular - // TODO: load actual most popular package and highest-rated screenshot and cache it - // TODO: link to the package page - a.black href=package_path(@most_popular_package.name) - img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false) - p - strong - =@most_popular_package.name - ' : - ' - =@most_popular_package.description - div.large-4.medium-12.small-12.columns.text-center - h2 Your help needed - // TODO: load actual most popular package without a screenshot and cache it - a href=upload_package_by_name_path('firefox') - img src="/images/dummy/no-screenshots-upload-one.svg" width="80%" - p - ' We lack a screenshot for the Firefox package. Can you - a href=upload_package_by_name_path('firefox') provide one - ' ? +/ div.row +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Newest upload +/ // TODO: load actual newest upload and cache it +/ // TODO: link to the package page +/ a.black href=package_path(@newest_uploaded_package.name) +/ img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false) +/ p +/ strong +/ =@newest_uploaded_package.name +/ ' : +/ ' +/ =@newest_uploaded_package.description +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Most popular +/ // TODO: load actual most popular package and highest-rated screenshot and cache it +/ // TODO: link to the package page +/ a.black href=package_path(@most_popular_package.name) +/ img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false) +/ p +/ strong +/ =@most_popular_package.name +/ ' : +/ ' +/ =@most_popular_package.description +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Your help needed +/ // TODO: load actual most popular package without a screenshot and cache it +/ a href=upload_package_by_name_path('firefox') +/ img src="/images/dummy/no-screenshots-upload-one.svg" width="80%" +/ p +/ ' We lack a screenshot for the Firefox package. Can you +/ a href=upload_package_by_name_path('firefox') provide one +/ ' ? From 13218841f4c07ff87c4937ab1a92ef2ceda0506f Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Mon, 20 Apr 2015 20:33:40 +0200 Subject: [PATCH 04/43] Optimisation: only query database if the package is not obviously blacklisted Performance gain: 20x --- lib/tasks/import_debian.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index dfd6c63..262ca02 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -74,11 +74,10 @@ task :import_debian => :environment do Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" - db_package = Package.find_by name: package[:Package] - if package_blacklisted? package # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE + db_package = Package.find_by name: package[:Package] if db_package Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" db_package.destroy @@ -88,6 +87,7 @@ task :import_debian => :environment do next end # if blacklisted + db_package = Package.find_by name: package[:Package] unless db_package Rails.logger.debug "No such package in our database." next From 1034afa92db59e69dcb6c432f48946e55e75265f Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 22 Apr 2015 21:45:06 +0200 Subject: [PATCH 05/43] Added "/" to URL paths when reading HTTP repositories --- lib/deb_importer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 05d3e89..8934793 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -17,7 +17,7 @@ module DebImporter # Load and parse a Release file of an APT repository def initialize(dist_url, components) @dist_url = dist_url - release_url = dist_url + "Release" + release_url = dist_url + "/Release" Rails.logger.debug "Loading Release file from #{release_url}" open(release_url) do |release_data| fields = get_fields(release_data) @@ -42,7 +42,7 @@ module DebImporter # and architecture (e.g. "amd64") def packages(component, architecture) # create path like "main/binary-amd64/Packages" - packages_path = "#{component}/binary-#{architecture}/Packages" + packages_path = "/#{component}/binary-#{architecture}/Packages" # Check if gzip or uncompressed files exist # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) for suffix in ['.gz', ''] From 8b8073bdd0b6023fb4935e3b0aae87c650d31739 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 22 Apr 2015 21:45:36 +0200 Subject: [PATCH 06/43] Added "debshots" namespace and list_deb_repos task --- lib/tasks/import_debian.rake | 147 ++++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 55 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 262ca02..9b2b9c1 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -39,72 +39,109 @@ BLACKLIST_SECTION_PATTERN = [ # Whether to delete a packag REMOVE_BLACKLISTED_PACKAGE = true -desc "Import package information from Debian-style Packages files" +namespace :debshots do + desc "Import/update package database from configured DEB repositories" -task :import_debian => :environment do - repositories = Rails.configuration.package_sources - Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::INFO - #Rails.logger.level = Logger::DEBUG + task :update_from_deb_repos => :environment do + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + #Rails.logger.level = Logger::DEBUG - Rails.logger.info "Importing Debian package information" + Rails.logger.info "Importing Debian package information" - repositories.each do |repository| - Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) - Rails.logger.info "> Supported architectures are: #{release.architectures}" - Rails.logger.info "> Supported components are: #{release.components}" + repositories.each do |repository| + Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" + release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "> Supported architectures are: #{release.architectures}" + Rails.logger.info "> Supported components are: #{release.components}" - # TODO: Remember what we imported to show it on the /about page + # TODO: Remember what we imported to show it on the /about page - release.components.split.each do |component| - Rails.logger.info "> Component: #{component}" - release.architectures.split.each do |architecture| - Rails.logger.info ">> Architecture: #{architecture}" - # Check if this component and architecture is available on this mirror - packages = release.packages(component, architecture) - unless packages - Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" - next - end - - Rails.logger.info "Packages file for #{component} on #{architecture} found." - Rails.logger.debug "Got: #{packages}" - packages.each do |package| - Rails.logger.info "> Package: #{package[:Package]}" - #Rails.logger.debug "Fetching package informaton from the database" - - if package_blacklisted? package - # Should the package get removed from the database? - if REMOVE_BLACKLISTED_PACKAGE - db_package = Package.find_by name: package[:Package] - if db_package - Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" - db_package.destroy - end - end - - next - end # if blacklisted - - db_package = Package.find_by name: package[:Package] - unless db_package - Rails.logger.debug "No such package in our database." + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.architectures.split.each do |architecture| + Rails.logger.info ">> Architecture: #{architecture}" + # Check if this component and architecture is available on this mirror + packages = release.packages(component, architecture) + unless packages + Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" next end - Rails.logger.debug "Found in database: #{db_package.id}" - update_data(package, db_package) - end # package.each + Rails.logger.info "Packages file for #{component} on #{architecture} found." + Rails.logger.debug "Got: #{packages}" + packages.each do |package| + Rails.logger.info "> Package: #{package[:Package]}" + #Rails.logger.debug "Fetching package informaton from the database" - end # architectures.each - end # components.each + if package_blacklisted? package + # Should the package get removed from the database? + if REMOVE_BLACKLISTED_PACKAGE + db_package = Package.find_by name: package[:Package] + if db_package + Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" + db_package.destroy + end + end - Rails.logger.info "Done parsing #{repository[:url]} repository." - Rails.logger.info "--------------------------------------------" - end # repositories.each + next + end # if blacklisted -end # task + db_package = Package.find_by name: package[:Package] + unless db_package + Rails.logger.debug "No such package in our database." + next + end + + Rails.logger.debug "Found in database: #{db_package.id}" + update_data(package, db_package) + end # package.each + + end # architectures.each + end # components.each + + Rails.logger.info "Done parsing #{repository[:url]} repository." + Rails.logger.info "--------------------------------------------" + end # repositories.each + + end # task + + + desc "List configured DEB repositories" + + task :list_deb_repos => :environment do + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + + Rails.logger.info "Listing configured DEB repositories" + + repositories.each do |repository| + Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" + release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "> Supported architectures are: #{release.architectures}" + Rails.logger.info "> Supported components are: #{release.components}" + + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.architectures.split.each do |architecture| + Rails.logger.info ">> Architecture: #{architecture}" + # Check if this component and architecture is available on this mirror + packages = release.packages(component, architecture) + if packages + Rails.logger.info "#{packages.count} packages found." + else + Rails.logger.info "No packages." + end + end # architectures.each + end # components.each + + end # repositories.each + + end # task + +end # namespace # Update the information about a package in the database def update_data(package, db_package) From d888829ce6d48124abb5e0283c5a23e701d546c3 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 22 Apr 2015 23:02:20 +0200 Subject: [PATCH 07/43] Added /about hint on missing packages --- app/views/welcome/about.slim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/views/welcome/about.slim b/app/views/welcome/about.slim index c522fb3..db7df42 100644 --- a/app/views/welcome/about.slim +++ b/app/views/welcome/about.slim @@ -86,6 +86,16 @@ ' If you would like to get a complete list of packages or screenshots in JSON ' format then contact us and we will give you the URL. + h2 Some packages are not found + + p + ' That's right. We import the package repositories from various Linux + ' distributions once a day. However we already filter out packages that + ' we think would not be able to have a meaningful screenshot. + ' Like libraries or documentation or developer packages. So it may happen + ' that you a package is not found on this web sites although it may be + ' available in your favorite Linux distribution. + h2 Uploading new screenshots p From 4c43e10ab3ddf12d8bac5483c191ade5afdda0f7 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 22 Apr 2015 23:03:25 +0200 Subject: [PATCH 08/43] Added architectures whitelist for Debian repository importer Only the given architectures are considered. This reduces the amount of computing required when updating the database from Debian repositories. And it probably limits the packages to those being availble on the more popular desktop architectures like amd64 and i386. --- config/environments/development.rb | 3 ++- config/environments/production.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/environments/development.rb b/config/environments/development.rb index b39331a..ced472c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -34,10 +34,11 @@ Debshots::Application.configure do # URL prefix leading to the static images that should get delivered by the web server config.images_path_prefix = '/screenshots' - # DEB package repositories to parse when running "rake import_debian" + # DEB package repositories to parse when running "rake debshots:... tasks" config.package_sources = [ { description: 'Development test files', type: 'apt', url: 'lib/tasks/files/', + architectures: ['i386', 'amd64'] #components: ['main','restricted','universe','multiverse'] } ] diff --git a/config/environments/production.rb b/config/environments/production.rb index 0bdc398..1743d4f 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -87,6 +87,7 @@ Debshots::Application.configure do config.package_sources = [ { description: 'Debian Unstable (Sid)', type: 'apt', url: 'http://ftp.de.debian.org/debian/dists/sid', + architectures: ['i386' ,'amd64'] #components: ['main','restricted','universe','multiverse'] } ] From beeb4cd37a813c3445bfe05f4687f229e7382395 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 22 Apr 2015 23:03:49 +0200 Subject: [PATCH 09/43] Stats counter added. Limiting to wanted archs. --- lib/tasks/import_debian.rake | 41 ++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 9b2b9c1..ab86a42 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -36,13 +36,18 @@ BLACKLIST_SECTION_PATTERN = [ /\/?cli-mono$/, ] -# Whether to delete a packag +# Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true namespace :debshots do desc "Import/update package database from configured DEB repositories" task :update_from_deb_repos => :environment do + # Counters for added, updated or removed packages from the database + stats_added = 0 + stats_updated = 0 + stats_removed = 0 + repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO @@ -51,9 +56,13 @@ namespace :debshots do Rails.logger.info "Importing Debian package information" repositories.each do |repository| + wanted_architectures = repository[:architectures] Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url], repository[:components]) Rails.logger.info "> Supported architectures are: #{release.architectures}" + if wanted_architectures + Rails.logger.info "> We only want architectures: #{wanted_architectures}" + end Rails.logger.info "> Supported components are: #{release.components}" # TODO: Remember what we imported to show it on the /about page @@ -61,6 +70,10 @@ namespace :debshots do release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.architectures.split.each do |architecture| + if wanted_architectures and not wanted_architectures.include?(architecture) + Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." + next + end Rails.logger.info ">> Architecture: #{architecture}" # Check if this component and architecture is available on this mirror packages = release.packages(component, architecture) @@ -82,6 +95,7 @@ namespace :debshots do if db_package Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" db_package.destroy + stats_removed +=1 end end @@ -90,42 +104,57 @@ namespace :debshots do db_package = Package.find_by name: package[:Package] unless db_package - Rails.logger.debug "No such package in our database." - next + Rails.logger.debug "No such package in our database. Creating one." + db_package = Package.create + stats_added += 1 end - Rails.logger.debug "Found in database: #{db_package.id}" + # Rails.logger.debug "Found in database: #{db_package}" update_data(package, db_package) + stats_updated += 1 end # package.each end # architectures.each end # components.each Rails.logger.info "Done parsing #{repository[:url]} repository." + Rails.logger.info "#{stats_added} packages added" + Rails.logger.info "#{stats_updated} packages updated" + Rails.logger.info "#{stats_removed} packages removed" Rails.logger.info "--------------------------------------------" end # repositories.each end # task - + # TODO: Some code duplication. Should be refactord. desc "List configured DEB repositories" task :list_deb_repos => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::INFO + Rails.logger.level = Logger::DEBUG Rails.logger.info "Listing configured DEB repositories" repositories.each do |repository| + wanted_architectures = repository[:architectures] Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url], repository[:components]) Rails.logger.info "> Supported architectures are: #{release.architectures}" + if wanted_architectures + Rails.logger.info "> We only want architectures: #{wanted_architectures}" + end Rails.logger.info "> Supported components are: #{release.components}" release.components.split.each do |component| Rails.logger.info "> Component: #{component}" release.architectures.split.each do |architecture| + if wanted_architectures and not wanted_architectures.include?(architecture) + Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." + next + end + Rails.logger.info ">> Architecture: #{architecture}" # Check if this component and architecture is available on this mirror packages = release.packages(component, architecture) From a15b2b6674953d060fa00d30ed5c2a8e716da321 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 00:04:06 +0200 Subject: [PATCH 10/43] DEB importer can now handle bzip2 files --- Gemfile | 3 +++ Gemfile.lock | 7 +++++++ lib/deb_importer.rb | 15 +++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 50a9289..ad00aa2 100644 --- a/Gemfile +++ b/Gemfile @@ -93,3 +93,6 @@ gem "paperclip", "~> 4.2" # Comply with stupid european cookie law gem 'cookies_eu' + +# Requires: apt-get install libbz2-dev +gem 'bzip2-ruby', :git => 'https://github.com/chewi/bzip2-ruby.git' diff --git a/Gemfile.lock b/Gemfile.lock index 8c7c9a0..09d4f11 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,9 @@ +GIT + remote: https://github.com/chewi/bzip2-ruby.git + revision: 92d492c2b40c32a5c039314666655ebc2f52a817 + specs: + bzip2-ruby (0.2.7) + GEM remote: https://rubygems.org/ specs: @@ -191,6 +197,7 @@ DEPENDENCIES better_errors binding_of_caller byebug + bzip2-ruby! coffee-rails (~> 4.1.0) cookies_eu fancybox2-rails (~> 0.2.8) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 8934793..3f6607c 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -1,5 +1,7 @@ # Various helper methods to update the database of packages +require 'bzip2' + module DebImporter # This module imports information about packages of a Linux distribution @@ -45,15 +47,24 @@ module DebImporter packages_path = "/#{component}/binary-#{architecture}/Packages" # Check if gzip or uncompressed files exist # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) - for suffix in ['.gz', ''] + for suffix in ['.bz2', '.gz', ''] packages_path_with_suffix = packages_path+suffix Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" url = @dist_url + packages_path_with_suffix begin Rails.logger.debug "Try opening URL: #{url}" - file = open(url) + begin + file = open(url) + puts "URL: #{url}" + puts "File: #{file}" + rescue OpenURI::HTTPError => e + Rails.logger.info "URL #{url} lead to #{e}. skipping." + next + end if suffix == '.gz' file = Zlib::GzipReader.new(file) + elsif suffix == '.bz2' + file = Bzip2::Reader.new(file) end Rails.logger.debug "File containing packages is: #{file}" #paragraphs = get_paragraphs(file) From c05b52895dbd4e09d40c708061ffda01da40176e Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 00:29:30 +0200 Subject: [PATCH 11/43] Parsing of i18n file added. Allows us to read long descriptions and add them to the package information --- lib/deb_importer.rb | 28 ++++++++++++++++++++++++++-- lib/tasks/import_debian.rake | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 3f6607c..b0aeb67 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -40,6 +40,32 @@ module DebImporter end # open end # def initialize + # TODO: Avoid code duplication (bz2, gz, ...) + def i18n(component, language) + for suffix in ['.bz2', '.gz', ''] + url = "#{@dist_url}/#{component}/i18n/Translation-en#{suffix}" + Rails.logger.debug "Looking for translations file: #{url}" + begin + Rails.logger.debug "Try opening URL: #{url}" + begin + file = open(url) + rescue OpenURI::HTTPError => e + Rails.logger.info "URL #{url} lead to #{e}. skipping." + next + end + if suffix == '.gz' + file = Zlib::GzipReader.new(file) + elsif suffix == '.bz2' + file = Bzip2::Reader.new(file) + end + Rails.logger.debug "File containing translations is: #{file}" + return get_paragraphs(file) + rescue Errno::ENOENT + Rails.logger.debug "URL could not be opened. Skipping." + end + end + end + # Try to load the Packages file for a certain component (e.g. "main") # and architecture (e.g. "amd64") def packages(component, architecture) @@ -55,8 +81,6 @@ module DebImporter Rails.logger.debug "Try opening URL: #{url}" begin file = open(url) - puts "URL: #{url}" - puts "File: #{file}" rescue OpenURI::HTTPError => e Rails.logger.info "URL #{url} lead to #{e}. skipping." next diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index ab86a42..0b02301 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -126,6 +126,42 @@ namespace :debshots do end # task + desc 'Update long description from i18n file' + + task :update_longdescription_from_deb_repos => :environment do + + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + Rails.logger.info "Importing long descriptoin from Debian repository (i18n)" + + repositories.each do |repository| + Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" + release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "> Supported components are: #{release.components}" + + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.i18n(component, 'en').each do |pkg| + Rails.logger.debug "i18n information: #{pkg}" + # See if we have that package in the database + if db_pkg = Package.find_by(name: pkg[:Package]) + Rails.logger.info "Updating long description for package #{db_pkg.name}" + db_pkg.long_description = pkg[:'Description-en'] + db_pkg.save + end + end # pkg.each + end # components.each + + + Rails.logger.info "Done updating long descriptions from translation file." + Rails.logger.info "--------------------------------------------" + end # repositories.each + + end # task + # TODO: Some code duplication. Should be refactord. desc "List configured DEB repositories" From 4c126a4ed09690b3306256220dfc05303c2aa48f Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 14:58:30 +0200 Subject: [PATCH 12/43] Added spring-pimped start scripts for rake, rails etc. --- bin/rails | 4 ++++ bin/rake | 4 ++++ bin/rspec | 7 +++++++ bin/spring | 15 +++++++++++++++ 4 files changed, 30 insertions(+) create mode 100755 bin/rspec create mode 100755 bin/spring diff --git a/bin/rails b/bin/rails index 728cd85..7feb6a3 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,8 @@ #!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end APP_PATH = File.expand_path('../../config/application', __FILE__) require_relative '../config/boot' require 'rails/commands' diff --git a/bin/rake b/bin/rake index 1724048..8017a02 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,8 @@ #!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/rspec b/bin/rspec new file mode 100755 index 0000000..20060eb --- /dev/null +++ b/bin/rspec @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end +require 'bundler/setup' +load Gem.bin_path('rspec-core', 'rspec') diff --git a/bin/spring b/bin/spring new file mode 100755 index 0000000..7b45d37 --- /dev/null +++ b/bin/spring @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby + +# This file loads spring without using Bundler, in order to be fast. +# It gets overwritten when you run the `spring binstub` command. + +unless defined?(Spring) + require "rubygems" + require "bundler" + + if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m) + Gem.paths = { "GEM_PATH" => [Bundler.bundle_path.to_s, *Gem.path].uniq } + gem "spring", match[1] + require "spring/binstub" + end +end From 7005c5397a5bf2743089da2a2af7ed0b10275961 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 14:59:23 +0200 Subject: [PATCH 13/43] Guard added --- Gemfile | 6 ++++++ Gemfile.lock | 48 +++++++++++++++++++++++++++++++++++++++++++++ Guardfile | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 Guardfile diff --git a/Gemfile b/Gemfile index ad00aa2..0c3a223 100644 --- a/Gemfile +++ b/Gemfile @@ -67,6 +67,12 @@ group :development, :test do # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' + + # Helps run tests automatically after changes + # TODO: not sure all these packages are required (http://buildingrails.com/a/rails_automated_testing_setup_for_beginners) + gem 'guard-rails' + gem 'minitest-rails' + gem 'guard-minitest' end # Zurb Foundation diff --git a/Gemfile.lock b/Gemfile.lock index 09d4f11..7c5f27c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -52,6 +52,8 @@ GEM builder (3.2.2) byebug (4.0.5) columnize (= 0.9.0) + celluloid (0.16.0) + timers (~> 4.0.0) choice (0.1.7) climate_control (0.0.3) activesupport (>= 3.0) @@ -74,11 +76,30 @@ GEM execjs (2.5.2) fancybox2-rails (0.2.8) railties (>= 3.1.0, < 5.0) + ffi (1.9.8) + formatador (0.2.5) foundation-rails (5.5.1.2) railties (>= 3.1.0) sass (>= 3.3.0, < 3.5) globalid (0.3.5) activesupport (>= 4.1.0) + guard (2.12.5) + formatador (>= 0.2.4) + listen (~> 2.7) + lumberjack (~> 1.0) + nenv (~> 0.1) + notiffany (~> 0.0) + pry (>= 0.9.12) + shellany (~> 0.0) + thor (>= 0.18.1) + guard-compat (1.2.1) + guard-minitest (2.4.4) + guard-compat (~> 1.2) + minitest (>= 3.0) + guard-rails (0.7.1) + guard (~> 2.11) + guard-compat (~> 1.0) + hitimes (1.2.2) i18n (0.7.0) jbuilder (2.2.13) activesupport (>= 3.0.0, < 5) @@ -90,16 +111,29 @@ GEM railties (>= 4.2.0) thor (>= 0.14, < 2.0) json (1.8.2) + listen (2.10.0) + celluloid (~> 0.16.0) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9) loofah (2.0.1) nokogiri (>= 1.5.9) + lumberjack (1.0.9) mail (2.6.3) mime-types (>= 1.16, < 3) + method_source (0.8.2) mime-types (2.4.3) mini_portile (0.6.2) minitest (5.6.0) + minitest-rails (2.1.1) + minitest (~> 5.4) + railties (~> 4.1) multi_json (1.11.0) + nenv (0.2.0) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) + notiffany (0.0.6) + nenv (~> 0.1) + shellany (~> 0.0) paperclip (4.2.1) activemodel (>= 3.0.0) activesupport (>= 3.0.0) @@ -110,6 +144,10 @@ GEM activerecord (>= 3.1) activesupport (>= 3.1) arel + pry (0.10.1) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) rack (1.6.0) rack-test (0.6.3) rack (>= 1.0) @@ -143,6 +181,9 @@ GEM rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) rake (10.4.2) + rb-fsevent (0.9.4) + rb-inotify (0.9.5) + ffi (>= 0.5.0) rdoc (4.2.0) json (~> 1.4) ruby-graphviz (1.0.9) @@ -156,6 +197,7 @@ GEM sdoc (0.4.1) json (~> 1.7, >= 1.7.7) rdoc (~> 4.0) + shellany (0.0.1) slim (3.0.3) temple (~> 0.7.3) tilt (>= 1.3.3, < 2.1) @@ -165,6 +207,7 @@ GEM activesupport (>= 3.1, < 5.0) railties (>= 3.1, < 5.0) slim (~> 3.0) + slop (3.6.0) spring (1.3.4) sprockets (3.0.1) rack (~> 1.0) @@ -176,6 +219,8 @@ GEM thor (0.19.1) thread_safe (0.3.5) tilt (1.4.1) + timers (4.0.1) + hitimes tzinfo (1.2.2) thread_safe (~> 0.1) uglifier (2.7.1) @@ -202,8 +247,11 @@ DEPENDENCIES cookies_eu fancybox2-rails (~> 0.2.8) foundation-rails + guard-minitest + guard-rails jbuilder (~> 2.0) jquery-rails + minitest-rails paperclip (~> 4.2) pg pg_search diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..0623256 --- /dev/null +++ b/Guardfile @@ -0,0 +1,55 @@ +# A sample Guardfile +# More info at https://github.com/guard/guard#readme + +## Uncomment and set this to only include directories you want to watch +# directories %w(app lib config test spec features) + +## Uncomment to clear the screen before every task +# clearing :on + +## Guard internally checks for changes in the Guardfile and exits. +## If you want Guard to automatically start up again, run guard in a +## shell loop, e.g.: +## +## $ while bundle exec guard; do echo "Restarting Guard..."; done +## +## Note: if you are using the `directories` clause above and you are not +## watching the project directory ('.'), then you will want to move +## the Guardfile to a watched dir and symlink it back, e.g. +# +# $ mkdir config +# $ mv Guardfile config/ +# $ ln -s config/Guardfile . +# +# and, you'll have to watch "config/Guardfile" instead of "Guardfile" + +# Guard-Rails supports a lot options with default values: +# daemon: false # runs the server as a daemon. +# debugger: false # enable ruby-debug gem. +# environment: 'development' # changes server environment. +# force_run: false # kills any process that's holding the listen port before attempting to (re)start Rails. +# pid_file: 'tmp/pids/[RAILS_ENV].pid' # specify your pid_file. +# host: 'localhost' # server hostname. +# port: 3000 # server port number. +# root: '/spec/dummy' # Rails' root path. +# server: thin # webserver engine. +# start_on_start: true # will start the server when starting Guard. +# timeout: 30 # waits untill restarting the Rails server, in seconds. +# zeus_plan: server # custom plan in zeus, only works with `zeus: true`. +# zeus: false # enables zeus gem. +# CLI: 'rails server' # customizes runner command. Omits all options except `pid_file`! + +guard 'rails' do + watch('Gemfile.lock') + watch(%r{^(config|lib)/.*}) +end + +guard :minitest do + watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" } + watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' } + watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" } + watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" } + watch(%r{^lib/(.+)\.rb$}) { |m| "test/lib/#{m[1]}_test.rb" } + watch(%r{^test/.+_test\.rb$}) + watch(%r{^test/test_helper\.rb$}) { 'test' } +end From 754953a653e1f2f5307e6d4f3a1d27d97a9e3176 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:18:55 +0200 Subject: [PATCH 14/43] Remove Rails 4 compatibility cruft. --- config/environments/production.rb | 4 +++- config/environments/test.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index 1743d4f..a5f1794 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -10,6 +10,9 @@ Debshots::Application.configure do # Rake tasks automatically ignore this option for performance. config.eager_load = true + # ActiveRecord errors propagate normally (forward deprecation to Rails 5) + config.active_record.raise_in_transactional_callbacks = true + # Full error reports are disabled and caching is turned on. config.consider_all_requests_local = false config.action_controller.perform_caching = true @@ -20,7 +23,6 @@ Debshots::Application.configure do # config.action_dispatch.rack_cache = true # Disable Rails's static asset server (Apache or nginx will already do this). - # config.serve_static_assets = false config.serve_static_files = true # Compress JavaScripts and CSS. diff --git a/config/environments/test.rb b/config/environments/test.rb index bf7a212..ee99f2f 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -13,7 +13,7 @@ Debshots::Application.configure do config.eager_load = false # Configure static asset server for tests with Cache-Control for performance. - config.serve_static_assets = true + config.serve_static_files = true config.static_cache_control = "public, max-age=3600" # Show full error reports and disable caching. From f741112818dc19bff5f1afe1082bd9e198e7ecc9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:19:11 +0200 Subject: [PATCH 15/43] Added migration to make package.long_description a longer text field --- .../20150422223156_change_package_longdescription_to_text.rb | 5 +++++ db/schema.rb | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20150422223156_change_package_longdescription_to_text.rb diff --git a/db/migrate/20150422223156_change_package_longdescription_to_text.rb b/db/migrate/20150422223156_change_package_longdescription_to_text.rb new file mode 100644 index 0000000..80ac3a4 --- /dev/null +++ b/db/migrate/20150422223156_change_package_longdescription_to_text.rb @@ -0,0 +1,5 @@ +class ChangePackageLongdescriptionToText < ActiveRecord::Migration + def change + change_column :packages, :long_description, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 489ff6c..61b2333 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150103194230) do +ActiveRecord::Schema.define(version: 20150422223156) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -31,7 +31,7 @@ ActiveRecord::Schema.define(version: 20150103194230) do t.string "maintainer_email", limit: 100 t.string "homepage", limit: 400 t.string "version", limit: 200 - t.string "long_description", limit: 2000 + t.text "long_description" t.string "origin", limit: 80 t.datetime "created_at" t.datetime "updated_at" From 63e3c623562cd3b621c417fe3368a37e68a8adbb Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:19:51 +0200 Subject: [PATCH 16/43] Importer tries to read long descriptions from translations file. Still buggy. The description text is not properly split after the first line. --- lib/tasks/import_debian.rake | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 0b02301..7021600 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -133,10 +133,19 @@ namespace :debshots do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO - # Rails.logger.level = Logger::DEBUG + Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing long descriptoin from Debian repository (i18n)" +# BUG +# Description-en: autoconf like tool +# ACR is an autoconf like tool that allows you to create configure scripts for +# your programs. The main aim of this tool is to teach developers how to create +# portable builds of their tools, just using generic functions wrapped by acr to +# generate portable shellscript. + +# Parses as "your programs. The ..." + repositories.each do |repository| Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url], repository[:components]) @@ -146,10 +155,13 @@ namespace :debshots do Rails.logger.info "> Component: #{component}" release.i18n(component, 'en').each do |pkg| Rails.logger.debug "i18n information: #{pkg}" + next unless pkg[:'Description-en'] # See if we have that package in the database if db_pkg = Package.find_by(name: pkg[:Package]) Rails.logger.info "Updating long description for package #{db_pkg.name}" - db_pkg.long_description = pkg[:'Description-en'] + text = pkg[:'Description-en'] + short_description, long_description = text.split("\n", 2) + db_pkg.long_description = long_description db_pkg.save end end # pkg.each From ed99d198f026c94e56d75c4cd6cebf6bced17f3e Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:20:06 +0200 Subject: [PATCH 17/43] Fixed controller tests --- test/controllers/packages_controller_test.rb | 26 ++++++------- test/controllers/welcome_controller_test.rb | 3 +- test/fixtures/packages.yml | 40 ++++++++++---------- test/fixtures/screenshots.yml | 36 +++++++++--------- 4 files changed, 52 insertions(+), 53 deletions(-) diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index f78cc4e..b3bcc38 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -2,23 +2,23 @@ require 'test_helper' class PackagesControllerTest < ActionController::TestCase test "should get index" do - get :index + get :grid assert_response :success end - test "should get with_screenshots" do - get :with_screenshots - assert_response :success - end + # test "should get with_screenshots" do + # get :with_screenshots + # assert_response :success + # end - test "should get without_screenshots" do - get :without_screenshots - assert_response :success - end + # test "should get without_screenshots" do + # get :without_screenshots + # assert_response :success + # end - test "should get moderate" do - get :moderate - assert_response :success - end + # test "should get moderate" do + # get :moderate + # assert_response :success + # end end diff --git a/test/controllers/welcome_controller_test.rb b/test/controllers/welcome_controller_test.rb index dff8e9d..23d1cc7 100644 --- a/test/controllers/welcome_controller_test.rb +++ b/test/controllers/welcome_controller_test.rb @@ -2,8 +2,7 @@ require 'test_helper' class WelcomeControllerTest < ActionController::TestCase test "should get index" do - get :index + get :home assert_response :success end - end diff --git a/test/fixtures/packages.yml b/test/fixtures/packages.yml index 0ac2d56..3b82496 100644 --- a/test/fixtures/packages.yml +++ b/test/fixtures/packages.yml @@ -1,23 +1,23 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html -one: - name: MyText - description: MyString - long_description: MyString - section: MyString - maintainer: MyString - maintainer_email: MyString - homepage: MyString - version: MyString - origin: MyString +# one: +# name: MyText +# description: MyString +# long_description: MyString +# section: MyString +# maintainer: MyString +# maintainer_email: MyString +# homepage: MyString +# version: MyString +# origin: MyString -two: - name: MyText - description: MyString - long_description: MyString - section: MyString - maintainer: MyString - maintainer_email: MyString - homepage: MyString - version: MyString - origin: MyString +# two: +# name: MyText +# description: MyString +# long_description: MyString +# section: MyString +# maintainer: MyString +# maintainer_email: MyString +# homepage: MyString +# version: MyString +# origin: MyString diff --git a/test/fixtures/screenshots.yml b/test/fixtures/screenshots.yml index 5ace412..b6e965a 100644 --- a/test/fixtures/screenshots.yml +++ b/test/fixtures/screenshots.yml @@ -1,21 +1,21 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html -one: - version: MyString - uploaddatetime: 2013-07-27 13:29:55 - uploaderhash: MyString - uploaderip: MyString - approved: - markedfordelete: - delete_reason: MyString - description: MyString +# one: +# version: MyString +# uploaddatetime: 2013-07-27 13:29:55 +# uploaderhash: MyString +# uploaderip: MyString +# approved: +# markedfordelete: +# delete_reason: MyString +# description: MyString -two: - version: MyString - uploaddatetime: 2013-07-27 13:29:55 - uploaderhash: MyString - uploaderip: MyString - approved: - markedfordelete: - delete_reason: MyString - description: MyString +# two: +# version: MyString +# uploaddatetime: 2013-07-27 13:29:55 +# uploaderhash: MyString +# uploaderip: MyString +# approved: +# markedfordelete: +# delete_reason: MyString +# description: MyString From 383f69c0b733a41dfd33bd7ed7367dabd2771bba Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:21:35 +0200 Subject: [PATCH 18/43] test_helper got updated automatically --- test/test_helper.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index bc7e05d..18b9705 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,11 +1,19 @@ -ENV["RAILS_ENV"] ||= "test" -require File.expand_path('../../config/environment', __FILE__) -require 'rails/test_help' +ENV["RAILS_ENV"] = "test" +require File.expand_path("../../config/environment", __FILE__) +require "rails/test_help" +#require "minitest/rails" + +# To add Capybara feature tests add `gem "minitest-rails-capybara"` +# to the test group in the Gemfile and uncomment the following: +# require "minitest/rails/capybara" + +# Uncomment for awesome colorful output +# require "minitest/pride" class ActiveSupport::TestCase - ActiveRecord::Migration.check_pending! + ActiveRecord::Migration.check_pending! - # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. + # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. # # Note: You'll currently still have to declare fixtures explicitly in integration tests # -- they do not yet inherit this setting From 9f6bcd12e484e6277007c9befe49e3959ec60502 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:21:55 +0200 Subject: [PATCH 19/43] Forward deprecation of tests to Rails 5 --- config/environments/test.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/environments/test.rb b/config/environments/test.rb index ee99f2f..1f815fc 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -12,6 +12,12 @@ Debshots::Application.configure do # preloads Rails for running tests, you may have to set it to true. config.eager_load = false + # ActiveRecord errors propagate normally (forward deprecation to Rails 5) + config.active_record.raise_in_transactional_callbacks = true + + # Test :sorted or :random. ":random" may be safer but harder to debug. + config.active_support.test_order = :sorted + # Configure static asset server for tests with Cache-Control for performance. config.serve_static_files = true config.static_cache_control = "public, max-age=3600" From 084c7cf32c150e34cc575801b95bbb101e4d522f Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:22:09 +0200 Subject: [PATCH 20/43] Using local dev and test databases. --- config/database.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/database.yml b/config/database.yml index 187ad98..1e0e165 100644 --- a/config/database.yml +++ b/config/database.yml @@ -7,10 +7,10 @@ development: #adapter: sqlite3 #database: db/development.sqlite3 adapter: postgresql - database: debshots + database: debshots_dev username: debshots - password: shootme - host: torf + password: GonwannEn0 + host: localhost pool: 5 timeout: 5000 @@ -19,10 +19,10 @@ development: # Do not set this db to the same as development or production. test: adapter: postgresql - database: debshots + database: debshots_test username: debshots - password: shootme - host: torf + password: GonwannEn0 + host: localhost pool: 5 timeout: 5000 From bab9c89ddb707c21cb45338c0fdb3cb69eeff9c0 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:22:26 +0200 Subject: [PATCH 21/43] Display long description in package detail page. --- app/views/packages/details.slim | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index 4904317..b6f0bcc 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -32,15 +32,9 @@ .bigpanel // Use action long description // TODO: Use actual description instead of placeholder text - p - ' Mail client suitable for free distribution. It supports different mail - ' accounts (POP, IMAP, Gmail), has an integrated learning Spam filter, and - ' offers easy organization of mails with tagging and virtual folders. Also, - ' more features can be added by installing extensions. - - p - ' The goal of Icedove is to produce a cross platform standalone mail - ' application using the XUL user interface language. + // TODO: split long_description by

tags + // TODO: What to do if long description is missing? + p =@package.long_description / a.button.round.expand href=upload_package_by_name_path(name: @package.name) Upload more screenshots for #{@package.name} From 0382c0936100480f7b38d3b092414380f11ca0d9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:22:48 +0200 Subject: [PATCH 22/43] Refactored packages controller. Tidied up a lot. Same functionality. --- app/controllers/packages_controller.rb | 53 ++++++++++++-------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index ae0836c..6611b26 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -1,39 +1,12 @@ class PackagesController < ApplicationController - def index - @packages = Package.includes(:screenshots) - - if params[:query] - @packages = @packages.text_search(params[:query]) - end - - unless params[:search].blank? - logger.debug "Searching for #{params[:search]}" - @packages = @packages.general_search(params[:search]) - end - - # Limit the packages to those that have approved screenshots. - # Also eager-load the screenshots. - if params[:show]=='onlywith' - @packages = @packages.where("screenshots.approved"=>true) - end - - if params[:show]=='with' - @packages = @packages.with_screenshots - logger.debug 'Limiting packages to those with screenshots' - elsif params[:show]=='without' - @packages = @packages.without_screenshots - logger.debug 'Limiting packages to those without screenshots' - end - end - def list - index + @packages = query_packages @packages = @packages.paginate(page: params[:page], per_page: 6) render 'packages/index-list.slim' end def grid - index + @packages = query_packages @packages = @packages.paginate(page: params[:page], per_page: 24) render 'packages/index-grid.slim' end @@ -99,4 +72,26 @@ class PackagesController < ApplicationController disposition: 'inline', status: 404 end + + # Return packages matching the criteria given by parameters + def query_packages + packages = Package.includes(:screenshots) + + # text search + if params[:search].present? + logger.debug "Searching for #{params[:search]}" + packages = packages.general_search(params[:search]) + end + + case params[:show] + when 'with' + packages = packages.with_screenshots + logger.debug 'Limiting packages to those with screenshots' + when 'without' + packages = packages.without_screenshots + logger.debug 'Limiting packages to those without screenshots' + end + + return packages + end end From bdfad36cc1d35755bbb662496f09539904ab89ad Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:22:59 +0200 Subject: [PATCH 23/43] Added hints on used components. --- README.Developer | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.Developer b/README.Developer index 1d21366..a01e1a2 100644 --- a/README.Developer +++ b/README.Developer @@ -34,4 +34,21 @@ Used: https://github.com/semaperepelitsa/jquery.fileupload-rails Which is a gem for: https://github.com/blueimp/jQuery-File-Upload Paperclip for attachment/screenshot file handlind within ActiveRecord -https://github.com/thoughtbot/paperclip \ No newline at end of file +https://github.com/thoughtbot/paperclip + +-- + +Components: + +- Rails 5 (widespread, up-to-date, solves many common problems) +- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search) +- Git (most people nowadays use Git) +- jQuery (well known and easy to use) +- Zurb Foundation (integrates nicer than Twitter Bootstrap) +- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails) + +-- + +Importer + +How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat From 369837a6d78780ed41facc0156783123c3b6a24a Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 17:23:26 +0200 Subject: [PATCH 24/43] Removed documentation on Elasticsearch. Elasticsearch is no longer planned for debshots. Full-text search using PostgreSQL is sufficient. --- README.rdoc | 73 ----------------------------------------------------- 1 file changed, 73 deletions(-) diff --git a/README.rdoc b/README.rdoc index cb3292d..5da0471 100644 --- a/README.rdoc +++ b/README.rdoc @@ -107,76 +107,3 @@ TODO /rss * Return an RSS feed of new uploads of screenshots - - -== Using Xapian - -Ruby gem "xapian" does not exist. Debian only has bindings for Ruby 1.9.x. -Xapian may be lightweight and nice but is too much hassle. - - - -== Elasticsearch - -= Install - -Get it from https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.6.deb - -Also: apt-get install openjdk-7-jre-headless - -= Configure - -Edit /etc/elasticsearch/elasticsearch.yml for single-server usage: -- cluster.name: ... -- node.name: ... -- index.number_of_shards: 1 -- index.number_of_replicas: 0 -- discovery.zen.ping.multicast.enabled: false - -= Add useful plugins - -/usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head -URL: serve:9200/_plugin/head - -/usr/share/elasticsearch/bin/plugin -install lukas-vlcek/bigdesk -URL: serve:9200/_plugin/bigdesk - -= Install River (that feeds PostgreSQL data into ElasticSearch): - -Get the PostgreSQL ODBC driver from -wget http://jdbc.postgresql.org/download/postgresql-9.3-1100.jdbc41.jar -(according to http://jdbc.postgresql.org/download.html) - -cp postgresql-9.3-1100.jdbc41.jar /usr/share/elasticsearch/plugins/river-jdbc - -/usr/share/elasticsearch/bin/plugin -install river-jdbc -url http://bit.ly/1iovWV9 -(according to https://github.com/jprante/elasticsearch-river-jdbc) - -Fix permissions after plugin installation: -chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins - -Restart Elasticsearch. - -curl -XPUT 'localhost:9200/_river/debshots_jdbc_river/_meta' -d '{ - "type" : "jdbc", - "jdbc" : { - "driver" : "org.postgresql.Driver", - "url" : "jdbc:postgresql://localhost:5432/debshots", - "user" : "debshots", - "password" : "shootme", - "sql" : "select id as _id,name,description from packages", - "strategy" : "simple", - "poll" : "5m" - }, - "index" : { - "index" : "debshots", - "type" : "jdbc" - } - }' - -Should return: - -{"ok":true,"_index":"_river","_type":"my_jdbc_river","_id":"_meta","_version":1} - - -TODO: Tire versus Rivers From 707d327765ff9995afb258d744d3851da8d08954 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 22:34:30 +0200 Subject: [PATCH 25/43] TODO added --- app/controllers/packages_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 6611b26..430c276 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -66,6 +66,7 @@ class PackagesController < ApplicationController private # Send a dummy thumbnail reading "No screenshot available. Sorry." + # TODO: Make sure it uses X-Sendfile correctly in production def thumbnail404 send_file Rails.root.join('public/images/dummy/no-screenshots-available.png'), type: "image/png", From 785191e2c29a10a97f3ec2723ef3bde8aeb8efbe Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 22:34:58 +0200 Subject: [PATCH 26/43] Minimal changes - nothing functional --- test/test_helper.rb | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/test/test_helper.rb b/test/test_helper.rb index 18b9705..6e9cc65 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,7 +1,8 @@ -ENV["RAILS_ENV"] = "test" +ENV['RAILS_ENV'] ||= 'test' require File.expand_path("../../config/environment", __FILE__) require "rails/test_help" -#require "minitest/rails" +# require "minitest/rails" +# require "minitest/autorun" # To add Capybara feature tests add `gem "minitest-rails-capybara"` # to the test group in the Gemfile and uncomment the following: @@ -11,13 +12,6 @@ require "rails/test_help" # require "minitest/pride" class ActiveSupport::TestCase - ActiveRecord::Migration.check_pending! - - # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. - # - # Note: You'll currently still have to declare fixtures explicitly in integration tests - # -- they do not yet inherit this setting - fixtures :all - - # Add more helper methods to be used by all tests here... + # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. + fixtures :all end From f9279337ed7b11af091d499b8c0544fe192c7520 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 22:35:15 +0200 Subject: [PATCH 27/43] Added unit test for Debian importer --- config/environments/test.rb | 16 ++++++++ test/lib/deb_importer_test.rb | 72 +++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 test/lib/deb_importer_test.rb diff --git a/config/environments/test.rb b/config/environments/test.rb index 1f815fc..510ab76 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -42,4 +42,20 @@ Debshots::Application.configure do # URL prefix leading to the static images that should get delivered by the web server config.images_path_prefix = '/screenshots' + + # DEB package repositories to parse when running "rake debshots:... tasks" + config.package_sources = [ + { + description: 'Development test files', type: 'apt', url: 'lib/tasks/files/', + architectures: ['i386', 'amd64'] + #components: ['main','restricted','universe','multiverse'] + } + ] + + config.images_path = Rails.root.join('public', 'screenshots') + + config.image_sizes = { + large: '800x600', + small: '160x120', + } end diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb new file mode 100644 index 0000000..3ef598c --- /dev/null +++ b/test/lib/deb_importer_test.rb @@ -0,0 +1,72 @@ +require 'open-uri' # allows to load URLs using open() +require 'deb_importer' +require 'test_helper' + +include DebImporter + +class PackagesHelperTest < ActionView::TestCase +# class PackagesHelperTest < MiniTest::Spec + + test "should be able to parse local Debian repository test files" do + + expected_package_counts = { + main: { + amd64: 8320, + i386: 8082 + }, + restricted: { + amd64: 21, + i386: 21 + }, + universe: { + amd64: 29406, + i386: 29471 + }, + multiverse: { + amd64: 672, + i386: 685 + } + } + + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + Rails.logger.info "Listing configured DEB repositories" + + repositories.each do |repository| + wanted_architectures = repository[:architectures] + Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" + release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "> Supported architectures are: #{release.architectures}" + if wanted_architectures + Rails.logger.info "> We only want architectures: #{wanted_architectures}" + end + Rails.logger.info "> Supported components are: #{release.components}" + + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.architectures.split.each do |architecture| + if wanted_architectures and not wanted_architectures.include?(architecture) + Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." + next + end + + Rails.logger.info ">> Architecture: #{architecture}" + + packages = release.packages(component, architecture) + packages_count = packages.count + + if packages + Rails.logger.info "#{packages_count} packages found." + assert_equal expected_package_counts[component.to_sym][architecture.to_sym], + packages_count + else + Rails.logger.info "No packages." + end + end # architectures.each + end # components.each + end # repositories.each + end # test +end # class From 15d56a53015b457e28c357845fad784a55a5f66c Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Thu, 23 Apr 2015 23:13:02 +0200 Subject: [PATCH 28/43] Load path added so that Guard can selective run this test --- test/lib/deb_importer_test.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index 3ef598c..a516704 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -1,3 +1,5 @@ +$LOAD_PATH << 'lib' + require 'open-uri' # allows to load URLs using open() require 'deb_importer' require 'test_helper' @@ -5,10 +7,11 @@ require 'test_helper' include DebImporter class PackagesHelperTest < ActionView::TestCase -# class PackagesHelperTest < MiniTest::Spec test "should be able to parse local Debian repository test files" do + # How many packages of each component and architecture we expect + # when parsing the text files in lib/tasks/files/... expected_package_counts = { main: { amd64: 8320, From f1059c520c021bff08d6be8684521f98d43d39b5 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 11:18:21 +0200 Subject: [PATCH 29/43] Refactoring the search for compress files (bz2, gz, ...) --- lib/deb_importer.rb | 64 ++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index b0aeb67..7d60196 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -42,26 +42,38 @@ module DebImporter # TODO: Avoid code duplication (bz2, gz, ...) def i18n(component, language) + url = "#{@dist_url}/#{component}/i18n/Translation-en" + file = find_and_open_compressed_url(url) + return get_paragraphs(file) + end + + # Look for the file or URL in various compressed formats + # (e.g. bz2, gz) and fall back to plain text format. + def find_and_open_compressed_url(base_url) + Rails.logger.debug "Looking for files at URL #{base_url} with different compressions" for suffix in ['.bz2', '.gz', ''] - url = "#{@dist_url}/#{component}/i18n/Translation-en#{suffix}" - Rails.logger.debug "Looking for translations file: #{url}" begin - Rails.logger.debug "Try opening URL: #{url}" begin + url = "#{base_url}#{suffix}" + Rails.logger.debug "Checking if file at #{url} is available" file = open(url) rescue OpenURI::HTTPError => e - Rails.logger.info "URL #{url} lead to #{e}. skipping." + Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." next end - if suffix == '.gz' - file = Zlib::GzipReader.new(file) - elsif suffix == '.bz2' + + # Decompress file depending on its filename suffix + case suffix + when '.bz2' file = Bzip2::Reader.new(file) + when '.gz' + file = Zlib::GzipReader.new(file) end Rails.logger.debug "File containing translations is: #{file}" - return get_paragraphs(file) + + return file rescue Errno::ENOENT - Rails.logger.debug "URL could not be opened. Skipping." + Rails.logger.error "URL #{url} could not be opened. Skipping." end end end @@ -70,36 +82,10 @@ module DebImporter # and architecture (e.g. "amd64") def packages(component, architecture) # create path like "main/binary-amd64/Packages" - packages_path = "/#{component}/binary-#{architecture}/Packages" - # Check if gzip or uncompressed files exist - # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) - for suffix in ['.bz2', '.gz', ''] - packages_path_with_suffix = packages_path+suffix - Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" - url = @dist_url + packages_path_with_suffix - begin - Rails.logger.debug "Try opening URL: #{url}" - begin - file = open(url) - rescue OpenURI::HTTPError => e - Rails.logger.info "URL #{url} lead to #{e}. skipping." - next - end - if suffix == '.gz' - file = Zlib::GzipReader.new(file) - elsif suffix == '.bz2' - file = Bzip2::Reader.new(file) - end - Rails.logger.debug "File containing packages is: #{file}" - #paragraphs = get_paragraphs(file) - #return paragraphs - #byebug - return get_paragraphs(file) - rescue Errno::ENOENT - Rails.logger.debug "URL could not be opened. Skipping." - end - end - return nil + packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages" + Rails.logger.debug "Loading packages from #{packages_path}" + file = find_and_open_compressed_url(packages_path) + return get_paragraphs(file) end # def packages end # class Release From ae66d11abb43fe9770bdf17edbbc5c70e4b30180 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 11:18:35 +0200 Subject: [PATCH 30/43] Minor reformatting --- test/lib/deb_importer_test.rb | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index a516704..e974ee6 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -16,21 +16,22 @@ class PackagesHelperTest < ActionView::TestCase main: { amd64: 8320, i386: 8082 - }, - restricted: { - amd64: 21, - i386: 21 - }, - universe: { - amd64: 29406, - i386: 29471 - }, - multiverse: { - amd64: 672, - i386: 685 - } - } + }, + restricted: { + amd64: 21, + i386: 21 + }, + universe: { + amd64: 29406, + i386: 29471 + }, + multiverse: { + amd64: 672, + i386: 685 + } + } + # Load information about test repository from environments/test.rb repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO @@ -57,16 +58,16 @@ class PackagesHelperTest < ActionView::TestCase end Rails.logger.info ">> Architecture: #{architecture}" - packages = release.packages(component, architecture) packages_count = packages.count if packages Rails.logger.info "#{packages_count} packages found." + assert_equal expected_package_counts[component.to_sym][architecture.to_sym], packages_count - else Rails.logger.info "No packages." + else end end # architectures.each end # components.each From 2a020a2193a0e3ded05741781f6d14e5f4c71cc1 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 12:19:56 +0200 Subject: [PATCH 31/43] Architectures and components limited for testing. --- config/environments/test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/environments/test.rb b/config/environments/test.rb index 510ab76..4f154be 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -47,8 +47,8 @@ Debshots::Application.configure do config.package_sources = [ { description: 'Development test files', type: 'apt', url: 'lib/tasks/files/', - architectures: ['i386', 'amd64'] - #components: ['main','restricted','universe','multiverse'] + architectures: ['amd64'], + components: ['main'] } ] From f0fec311236e6d2cdd80af311216c62b03d719a1 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 12:20:06 +0200 Subject: [PATCH 32/43] Removed "rails s" from Guardfile --- Guardfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Guardfile b/Guardfile index 0623256..bee49e7 100644 --- a/Guardfile +++ b/Guardfile @@ -39,10 +39,10 @@ # zeus: false # enables zeus gem. # CLI: 'rails server' # customizes runner command. Omits all options except `pid_file`! -guard 'rails' do - watch('Gemfile.lock') - watch(%r{^(config|lib)/.*}) -end +#guard 'rails' do +# watch('Gemfile.lock') +# watch(%r{^(config|lib)/.*}) +#end guard :minitest do watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" } From 8031edf2d13827ee1a46424fba5f4385ee5bb913 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 12:20:19 +0200 Subject: [PATCH 33/43] Tidying up --- lib/deb_importer.rb | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 7d60196..0d36d33 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -17,7 +17,7 @@ module DebImporter attr_reader :architectures, :components, :description, :codename, :origin, :version, :files # Load and parse a Release file of an APT repository - def initialize(dist_url, components) + def initialize(dist_url) @dist_url = dist_url release_url = dist_url + "/Release" Rails.logger.debug "Loading Release file from #{release_url}" @@ -29,18 +29,12 @@ module DebImporter @codename = fields[:Codename] @origin = fields[:Origin] @version = fields[:Version] - # Not needed at the moment. We do not check sizes or checksums yet. - #when 'SHA1', 'SHA256', 'MD5Sum' - # next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them - # @files = [] - # value.lines.each do |line| - # path = line.split.last - # @files << path - # end + + # TODO: check sizes and checksums end # open end # def initialize - # TODO: Avoid code duplication (bz2, gz, ...) + # Get package information from translation (i18n) files def i18n(component, language) url = "#{@dist_url}/#{component}/i18n/Translation-en" file = find_and_open_compressed_url(url) @@ -92,7 +86,7 @@ module DebImporter private - # Gather the fields of a Debian control file and return them as a hash + # Gather the fields of a Debian control file section and return them as a hash def get_fields(data) fields = {} From 4fa645d83059d0c610feefd5451ba24f74696ce3 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 12:20:35 +0200 Subject: [PATCH 34/43] Shortened repository importer test --- test/lib/deb_importer_test.rb | 49 +++++++++++------------------------ 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index e974ee6..5673803 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -31,46 +31,27 @@ class PackagesHelperTest < ActionView::TestCase } } - # Load information about test repository from environments/test.rb - repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::INFO + Rails.logger.level = Logger::ERROR + # Rails.logger.level = Logger::INFO # Rails.logger.level = Logger::DEBUG - Rails.logger.info "Listing configured DEB repositories" + # Load information about test repository from environments/test.rb + repositories = Rails.configuration.package_sources - repositories.each do |repository| - wanted_architectures = repository[:architectures] - Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) - Rails.logger.info "> Supported architectures are: #{release.architectures}" - if wanted_architectures - Rails.logger.info "> We only want architectures: #{wanted_architectures}" - end - Rails.logger.info "> Supported components are: #{release.components}" + Rails.logger.info "Test parsing local DEB-style repository" - release.components.split.each do |component| - Rails.logger.info "> Component: #{component}" - release.architectures.split.each do |architecture| - if wanted_architectures and not wanted_architectures.include?(architecture) - Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." - next - end + repository = repositories.first + release = DebImporter::Release.new(repository[:url]) + assert_equal release.components, 'main restricted universe multiverse' - Rails.logger.info ">> Architecture: #{architecture}" - packages = release.packages(component, architecture) - packages_count = packages.count + packages = release.packages('main', 'amd64') + assert_instance_of Enumerator, packages + first_package = packages.first + assert_equal 'account-plugin-aim', first_package[:Package] - if packages - Rails.logger.info "#{packages_count} packages found." - - assert_equal expected_package_counts[component.to_sym][architecture.to_sym], - packages_count - Rails.logger.info "No packages." - else - end - end # architectures.each - end # components.each - end # repositories.each + # .count must be checked seperately because it changes the position in the enumerator + packages = release.packages('main', 'amd64') + assert_equal 8320, packages.count end # test end # class From 061306e09ebe2733b76318c1f8f5df63f9c076eb Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 12:53:57 +0200 Subject: [PATCH 35/43] Tidied up --- test/lib/deb_importer_test.rb | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index 5673803..0a4cac4 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -9,28 +9,6 @@ include DebImporter class PackagesHelperTest < ActionView::TestCase test "should be able to parse local Debian repository test files" do - - # How many packages of each component and architecture we expect - # when parsing the text files in lib/tasks/files/... - expected_package_counts = { - main: { - amd64: 8320, - i386: 8082 - }, - restricted: { - amd64: 21, - i386: 21 - }, - universe: { - amd64: 29406, - i386: 29471 - }, - multiverse: { - amd64: 672, - i386: 685 - } - } - Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::ERROR # Rails.logger.level = Logger::INFO @@ -45,12 +23,13 @@ class PackagesHelperTest < ActionView::TestCase release = DebImporter::Release.new(repository[:url]) assert_equal release.components, 'main restricted universe multiverse' + # Packages enumerator must be restarted to start from position 1 packages = release.packages('main', 'amd64') assert_instance_of Enumerator, packages first_package = packages.first assert_equal 'account-plugin-aim', first_package[:Package] - # .count must be checked seperately because it changes the position in the enumerator + # Packages enumerator must be restarted to start from position 1 packages = release.packages('main', 'amd64') assert_equal 8320, packages.count end # test From c944e1ba913fefbe71056348e9ef638ab3453e8f Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 13:48:55 +0200 Subject: [PATCH 36/43] Fixed parser to add proper newlines in multi-line values Example: Key: Value1 Value2 Value3 Before: "Value1Value2\nValue" After: "Value1\nValue2\nValue" --- lib/deb_importer.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 0d36d33..914da70 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -34,7 +34,8 @@ module DebImporter end # open end # def initialize - # Get package information from translation (i18n) files + # Get package information from translation (i18n) files. + # Returns an enumerator of packages. def i18n(component, language) url = "#{@dist_url}/#{component}/i18n/Translation-en" file = find_and_open_compressed_url(url) @@ -96,12 +97,16 @@ module DebImporter when /^(.+): (.+)/ # "Key: Value" fields[name.to_sym]=value unless value.empty? name,value=$1,$2 - when /^(.+):\s*/ # "Key:" (start of multi-line entry) + when /^(.+):$/ # "Key:" (start of multi-line entry without value in line) fields[name.to_sym]=value unless value.empty? name=$1 value='' when /^\s+(.+)/ # " Indented multi-line value" - value << $1+"\n" + # Add a newline for multi-line entries ("Key: Value\n Foo\n Bar") + unless value.empty? + value << "\n" + end + value << $1 when /^\s+$/ # Empty line break end From 10ebc63dc17d8e917d84727f73d8d87311d27373 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 13:49:05 +0200 Subject: [PATCH 37/43] Added test for i18n long description --- test/lib/deb_importer_test.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb index 0a4cac4..cee922c 100644 --- a/test/lib/deb_importer_test.rb +++ b/test/lib/deb_importer_test.rb @@ -32,5 +32,9 @@ class PackagesHelperTest < ActionView::TestCase # Packages enumerator must be restarted to start from position 1 packages = release.packages('main', 'amd64') assert_equal 8320, packages.count + + # Get information from translation file + i18n = release.i18n('main', 'en') + assert_match /^Real-time strategy game of ancient warfare\n0 A\.D\./, i18n.first[:'Description-en'] end # test end # class From f511dfbd56bc6f9b0eb8fec8bfa16cf3a0109033 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 16:58:08 +0200 Subject: [PATCH 38/43] Handle missing files/URLs gracefully. If no version of a URL is found the importer just skips the URL now instead of throwing an error. --- lib/deb_importer.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 914da70..b85dba4 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -39,7 +39,11 @@ module DebImporter def i18n(component, language) url = "#{@dist_url}/#{component}/i18n/Translation-en" file = find_and_open_compressed_url(url) - return get_paragraphs(file) + if file + return get_paragraphs(file) + else + return [] + end end # Look for the file or URL in various compressed formats @@ -68,9 +72,12 @@ module DebImporter return file rescue Errno::ENOENT - Rails.logger.error "URL #{url} could not be opened. Skipping." + Rails.logger.debug "URL #{url} could not be opened. Skipping." end end + + Rails.logger.error "No file found at #{url} and various compression extensions." + return nil end # Try to load the Packages file for a certain component (e.g. "main") From 7c6b5c7205d8bb29524db8e03625aa98acd11642 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 16:59:47 +0200 Subject: [PATCH 39/43] Removed component argument from DebImporter::Release.new --- lib/tasks/import_debian.rake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 7021600..f79f580 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -57,8 +57,8 @@ namespace :debshots do repositories.each do |repository| wanted_architectures = repository[:architectures] - Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" + release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported architectures are: #{release.architectures}" if wanted_architectures Rails.logger.info "> We only want architectures: #{wanted_architectures}" @@ -148,7 +148,7 @@ namespace :debshots do repositories.each do |repository| Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) + release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported components are: #{release.components}" release.components.split.each do |component| @@ -187,8 +187,8 @@ namespace :debshots do repositories.each do |repository| wanted_architectures = repository[:architectures] - Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) + Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" + release = DebImporter::Release.new(repository[:url]) Rails.logger.info "> Supported architectures are: #{release.architectures}" if wanted_architectures Rails.logger.info "> We only want architectures: #{wanted_architectures}" From da6309400c0ad0a8cc2bde19ca25f25f55c94128 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 17:00:15 +0200 Subject: [PATCH 40/43] Refactored check if a package name or section is blacklisted --- lib/tasks/import_debian.rake | 70 ++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 39 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index f79f580..761f591 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -88,7 +88,7 @@ namespace :debshots do Rails.logger.info "> Package: #{package[:Package]}" #Rails.logger.debug "Fetching package informaton from the database" - if package_blacklisted? package + if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] # Should the package get removed from the database? if REMOVE_BLACKLISTED_PACKAGE db_package = Package.find_by name: package[:Package] @@ -133,19 +133,10 @@ namespace :debshots do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) Rails.logger.level = Logger::INFO - Rails.logger.level = Logger::DEBUG + # Rails.logger.level = Logger::DEBUG Rails.logger.info "Importing long descriptoin from Debian repository (i18n)" -# BUG -# Description-en: autoconf like tool -# ACR is an autoconf like tool that allows you to create configure scripts for -# your programs. The main aim of this tool is to teach developers how to create -# portable builds of their tools, just using generic functions wrapped by acr to -# generate portable shellscript. - -# Parses as "your programs. The ..." - repositories.each do |repository| Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" release = DebImporter::Release.new(repository[:url]) @@ -155,7 +146,13 @@ namespace :debshots do Rails.logger.info "> Component: #{component}" release.i18n(component, 'en').each do |pkg| Rails.logger.debug "i18n information: #{pkg}" - next unless pkg[:'Description-en'] + unless pkg[:'Description-en'] + Rails.logger.debug "No Description-en found in section." + next + end + if package_name_blacklisted? pkg[:Package] + next + end # See if we have that package in the database if db_pkg = Package.find_by(name: pkg[:Package]) Rails.logger.info "Updating long description for package #{db_pkg.name}" @@ -163,6 +160,8 @@ namespace :debshots do short_description, long_description = text.split("\n", 2) db_pkg.long_description = long_description db_pkg.save + else + Rails.logger.debug "Package not in database. Nothing to update." end end # pkg.each end # components.each @@ -180,8 +179,8 @@ namespace :debshots do task :list_deb_repos => :environment do repositories = Rails.configuration.package_sources Rails.logger = Logger.new(STDOUT) - # Rails.logger.level = Logger::INFO - Rails.logger.level = Logger::DEBUG + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG Rails.logger.info "Listing configured DEB repositories" @@ -240,33 +239,26 @@ def update_data(package, db_package) end db_package.save - - # TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en end -# Check if a package is blacklisted and should not be imported. +# Check if a package name is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. -# Argument: Hash containing package information from a Packages file -def package_blacklisted?(package) - Rails.logger.debug "Blacklist check of package #{package[:Package]}" - - # TODO: add a flag that allows to manually blacklist a package - - BLACKLIST_SECTION_PATTERN.each do |pattern| - Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}" - if package[:Section] =~ pattern - Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')" - return true - end +def package_name_blacklisted?(name) + if match=BLACKLIST_NAME_PATTERN.find { |pattern| name=~pattern} + Rails.logger.debug " > Blacklisted by name ('#{name}')" + return true + else + return false + end +end + +# Check if a package's section is blacklisted and should not be imported. +# This prevents importing boring software that likely does not get a useful screenshot. +def package_section_blacklisted?(section) + if match=BLACKLIST_SECTION_PATTERN.find { |pattern| section=~pattern} + Rails.logger.debug " > Blacklisted by section ('#{section}')" + return true + else + return false end - - BLACKLIST_NAME_PATTERN.each do |pattern| - Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}" - if package[:Package] =~ pattern - Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')" - return true - end - end - - false end From 9a0da2b396658243a66264a5f56d6fcb18bcc121 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 17:00:45 +0200 Subject: [PATCH 41/43] Unused variable removed --- lib/tasks/import_debian.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index 761f591..946eaf5 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -244,7 +244,7 @@ end # Check if a package name is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. def package_name_blacklisted?(name) - if match=BLACKLIST_NAME_PATTERN.find { |pattern| name=~pattern} + if BLACKLIST_NAME_PATTERN.find { |pattern| name=~pattern} Rails.logger.debug " > Blacklisted by name ('#{name}')" return true else @@ -255,7 +255,7 @@ end # Check if a package's section is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. def package_section_blacklisted?(section) - if match=BLACKLIST_SECTION_PATTERN.find { |pattern| section=~pattern} + if BLACKLIST_SECTION_PATTERN.find { |pattern| section=~pattern} Rails.logger.debug " > Blacklisted by section ('#{section}')" return true else From b421b9e0f6d1bd36535de62aaa35048657274255 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 17:54:23 +0200 Subject: [PATCH 42/43] Background color made darker to see the difference to white areas better. --- app/assets/stylesheets/my.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/stylesheets/my.scss b/app/assets/stylesheets/my.scss index 81e0468..abc4b79 100644 --- a/app/assets/stylesheets/my.scss +++ b/app/assets/stylesheets/my.scss @@ -1,7 +1,7 @@ // -------------- debshots --------------------------- -$page-background: #f2f2f2; +$page-background: #e8e8e8; $footer-background: #333333; $footer-text-color: #b3b3b3; From 96f3de022362dbb7ea4d80eb5f281e4a51d0b4ab Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Fri, 24 Apr 2015 17:54:37 +0200 Subject: [PATCH 43/43] Show placeholder image if a package has no screenshots. --- app/views/packages/details.slim | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index b6f0bcc..817766b 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -11,19 +11,20 @@ .small-7.medium-7.columns p.subtitle = @package.description - - - @package.screenshots.each do |screenshot| - .row.listview - .small-12.columns - a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption - = image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption) - .imgcaption =screenshot.caption - // TODO: Enable button only if admin or uploader - a.button.tiny.alert[ - href=delete_screenshot_path(@package.name, screenshot.id) - onclick="return confirm('Really delete the screenshot?');" - ] Delete screenshot - + - if @package.screenshots.count > 0 + - @package.screenshots.each do |screenshot| + .row.listview + .small-12.columns + a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption + = image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption) + .imgcaption =screenshot.caption + // TODO: Enable button only if admin or uploader + a.button.tiny.alert[ + href=delete_screenshot_path(@package.name, screenshot.id) + onclick="return confirm('Really delete the screenshot?');" + ] Delete screenshot + - else + img src='/images/dummy/no-screenshots-upload-one.svg' width='100%' // TODO: Enable comments in a later version // = partial '/package/comments'