diff --git a/Gemfile b/Gemfile index 50a9289..0c3a223 100644 --- a/Gemfile +++ b/Gemfile @@ -67,6 +67,12 @@ group :development, :test do # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' + + # Helps run tests automatically after changes + # TODO: not sure all these packages are required (http://buildingrails.com/a/rails_automated_testing_setup_for_beginners) + gem 'guard-rails' + gem 'minitest-rails' + gem 'guard-minitest' end # Zurb Foundation @@ -93,3 +99,6 @@ gem "paperclip", "~> 4.2" # Comply with stupid european cookie law gem 'cookies_eu' + +# Requires: apt-get install libbz2-dev +gem 'bzip2-ruby', :git => 'https://github.com/chewi/bzip2-ruby.git' diff --git a/Gemfile.lock b/Gemfile.lock index 8c7c9a0..7c5f27c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,9 @@ +GIT + remote: https://github.com/chewi/bzip2-ruby.git + revision: 92d492c2b40c32a5c039314666655ebc2f52a817 + specs: + bzip2-ruby (0.2.7) + GEM remote: https://rubygems.org/ specs: @@ -46,6 +52,8 @@ GEM builder (3.2.2) byebug (4.0.5) columnize (= 0.9.0) + celluloid (0.16.0) + timers (~> 4.0.0) choice (0.1.7) climate_control (0.0.3) activesupport (>= 3.0) @@ -68,11 +76,30 @@ GEM execjs (2.5.2) fancybox2-rails (0.2.8) railties (>= 3.1.0, < 5.0) + ffi (1.9.8) + formatador (0.2.5) foundation-rails (5.5.1.2) railties (>= 3.1.0) sass (>= 3.3.0, < 3.5) globalid (0.3.5) activesupport (>= 4.1.0) + guard (2.12.5) + formatador (>= 0.2.4) + listen (~> 2.7) + lumberjack (~> 1.0) + nenv (~> 0.1) + notiffany (~> 0.0) + pry (>= 0.9.12) + shellany (~> 0.0) + thor (>= 0.18.1) + guard-compat (1.2.1) + guard-minitest (2.4.4) + guard-compat (~> 1.2) + minitest (>= 3.0) + guard-rails (0.7.1) + guard (~> 2.11) + guard-compat (~> 1.0) + hitimes (1.2.2) i18n (0.7.0) jbuilder (2.2.13) activesupport (>= 3.0.0, < 5) @@ -84,16 +111,29 @@ GEM railties (>= 4.2.0) thor (>= 0.14, < 2.0) json (1.8.2) + listen (2.10.0) + celluloid (~> 0.16.0) + rb-fsevent (>= 0.9.3) + rb-inotify (>= 0.9) loofah (2.0.1) nokogiri (>= 1.5.9) + lumberjack (1.0.9) mail (2.6.3) mime-types (>= 1.16, < 3) + method_source (0.8.2) mime-types (2.4.3) mini_portile (0.6.2) minitest (5.6.0) + minitest-rails (2.1.1) + minitest (~> 5.4) + railties (~> 4.1) multi_json (1.11.0) + nenv (0.2.0) nokogiri (1.6.6.2) mini_portile (~> 0.6.0) + notiffany (0.0.6) + nenv (~> 0.1) + shellany (~> 0.0) paperclip (4.2.1) activemodel (>= 3.0.0) activesupport (>= 3.0.0) @@ -104,6 +144,10 @@ GEM activerecord (>= 3.1) activesupport (>= 3.1) arel + pry (0.10.1) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) rack (1.6.0) rack-test (0.6.3) rack (>= 1.0) @@ -137,6 +181,9 @@ GEM rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) rake (10.4.2) + rb-fsevent (0.9.4) + rb-inotify (0.9.5) + ffi (>= 0.5.0) rdoc (4.2.0) json (~> 1.4) ruby-graphviz (1.0.9) @@ -150,6 +197,7 @@ GEM sdoc (0.4.1) json (~> 1.7, >= 1.7.7) rdoc (~> 4.0) + shellany (0.0.1) slim (3.0.3) temple (~> 0.7.3) tilt (>= 1.3.3, < 2.1) @@ -159,6 +207,7 @@ GEM activesupport (>= 3.1, < 5.0) railties (>= 3.1, < 5.0) slim (~> 3.0) + slop (3.6.0) spring (1.3.4) sprockets (3.0.1) rack (~> 1.0) @@ -170,6 +219,8 @@ GEM thor (0.19.1) thread_safe (0.3.5) tilt (1.4.1) + timers (4.0.1) + hitimes tzinfo (1.2.2) thread_safe (~> 0.1) uglifier (2.7.1) @@ -191,12 +242,16 @@ DEPENDENCIES better_errors binding_of_caller byebug + bzip2-ruby! coffee-rails (~> 4.1.0) cookies_eu fancybox2-rails (~> 0.2.8) foundation-rails + guard-minitest + guard-rails jbuilder (~> 2.0) jquery-rails + minitest-rails paperclip (~> 4.2) pg pg_search diff --git a/Guardfile b/Guardfile new file mode 100644 index 0000000..bee49e7 --- /dev/null +++ b/Guardfile @@ -0,0 +1,55 @@ +# A sample Guardfile +# More info at https://github.com/guard/guard#readme + +## Uncomment and set this to only include directories you want to watch +# directories %w(app lib config test spec features) + +## Uncomment to clear the screen before every task +# clearing :on + +## Guard internally checks for changes in the Guardfile and exits. +## If you want Guard to automatically start up again, run guard in a +## shell loop, e.g.: +## +## $ while bundle exec guard; do echo "Restarting Guard..."; done +## +## Note: if you are using the `directories` clause above and you are not +## watching the project directory ('.'), then you will want to move +## the Guardfile to a watched dir and symlink it back, e.g. +# +# $ mkdir config +# $ mv Guardfile config/ +# $ ln -s config/Guardfile . +# +# and, you'll have to watch "config/Guardfile" instead of "Guardfile" + +# Guard-Rails supports a lot options with default values: +# daemon: false # runs the server as a daemon. +# debugger: false # enable ruby-debug gem. +# environment: 'development' # changes server environment. +# force_run: false # kills any process that's holding the listen port before attempting to (re)start Rails. +# pid_file: 'tmp/pids/[RAILS_ENV].pid' # specify your pid_file. +# host: 'localhost' # server hostname. +# port: 3000 # server port number. +# root: '/spec/dummy' # Rails' root path. +# server: thin # webserver engine. +# start_on_start: true # will start the server when starting Guard. +# timeout: 30 # waits untill restarting the Rails server, in seconds. +# zeus_plan: server # custom plan in zeus, only works with `zeus: true`. +# zeus: false # enables zeus gem. +# CLI: 'rails server' # customizes runner command. Omits all options except `pid_file`! + +#guard 'rails' do +# watch('Gemfile.lock') +# watch(%r{^(config|lib)/.*}) +#end + +guard :minitest do + watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" } + watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' } + watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" } + watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" } + watch(%r{^lib/(.+)\.rb$}) { |m| "test/lib/#{m[1]}_test.rb" } + watch(%r{^test/.+_test\.rb$}) + watch(%r{^test/test_helper\.rb$}) { 'test' } +end diff --git a/README.Developer b/README.Developer index 1d21366..a01e1a2 100644 --- a/README.Developer +++ b/README.Developer @@ -34,4 +34,21 @@ Used: https://github.com/semaperepelitsa/jquery.fileupload-rails Which is a gem for: https://github.com/blueimp/jQuery-File-Upload Paperclip for attachment/screenshot file handlind within ActiveRecord -https://github.com/thoughtbot/paperclip \ No newline at end of file +https://github.com/thoughtbot/paperclip + +-- + +Components: + +- Rails 5 (widespread, up-to-date, solves many common problems) +- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search) +- Git (most people nowadays use Git) +- jQuery (well known and easy to use) +- Zurb Foundation (integrates nicer than Twitter Bootstrap) +- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails) + +-- + +Importer + +How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat diff --git a/README.rdoc b/README.rdoc index cb3292d..5da0471 100644 --- a/README.rdoc +++ b/README.rdoc @@ -107,76 +107,3 @@ TODO /rss * Return an RSS feed of new uploads of screenshots - - -== Using Xapian - -Ruby gem "xapian" does not exist. Debian only has bindings for Ruby 1.9.x. -Xapian may be lightweight and nice but is too much hassle. - - - -== Elasticsearch - -= Install - -Get it from https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.6.deb - -Also: apt-get install openjdk-7-jre-headless - -= Configure - -Edit /etc/elasticsearch/elasticsearch.yml for single-server usage: -- cluster.name: ... -- node.name: ... -- index.number_of_shards: 1 -- index.number_of_replicas: 0 -- discovery.zen.ping.multicast.enabled: false - -= Add useful plugins - -/usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head -URL: serve:9200/_plugin/head - -/usr/share/elasticsearch/bin/plugin -install lukas-vlcek/bigdesk -URL: serve:9200/_plugin/bigdesk - -= Install River (that feeds PostgreSQL data into ElasticSearch): - -Get the PostgreSQL ODBC driver from -wget http://jdbc.postgresql.org/download/postgresql-9.3-1100.jdbc41.jar -(according to http://jdbc.postgresql.org/download.html) - -cp postgresql-9.3-1100.jdbc41.jar /usr/share/elasticsearch/plugins/river-jdbc - -/usr/share/elasticsearch/bin/plugin -install river-jdbc -url http://bit.ly/1iovWV9 -(according to https://github.com/jprante/elasticsearch-river-jdbc) - -Fix permissions after plugin installation: -chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins - -Restart Elasticsearch. - -curl -XPUT 'localhost:9200/_river/debshots_jdbc_river/_meta' -d '{ - "type" : "jdbc", - "jdbc" : { - "driver" : "org.postgresql.Driver", - "url" : "jdbc:postgresql://localhost:5432/debshots", - "user" : "debshots", - "password" : "shootme", - "sql" : "select id as _id,name,description from packages", - "strategy" : "simple", - "poll" : "5m" - }, - "index" : { - "index" : "debshots", - "type" : "jdbc" - } - }' - -Should return: - -{"ok":true,"_index":"_river","_type":"my_jdbc_river","_id":"_meta","_version":1} - - -TODO: Tire versus Rivers diff --git a/app/assets/stylesheets/my.scss b/app/assets/stylesheets/my.scss index 1c00472..9dc4e4d 100644 --- a/app/assets/stylesheets/my.scss +++ b/app/assets/stylesheets/my.scss @@ -1,7 +1,7 @@ // -------------- debshots --------------------------- -$page-background: #f2f2f2; +$page-background: #e8e8e8; $footer-background: #333333; $footer-text-color: #b3b3b3; @@ -148,6 +148,11 @@ html, body { { width: 100%; } + + img + { + box-shadow: 3px 3px 10px 0px #888888; + } } .row .gridview @@ -228,6 +233,7 @@ a.black .imgcaption { text-align: center; + font-variant: italic; } /* Status of a screenshot (e.g. if it has to be moderated) */ diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index ae0836c..430c276 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -1,39 +1,12 @@ class PackagesController < ApplicationController - def index - @packages = Package.includes(:screenshots) - - if params[:query] - @packages = @packages.text_search(params[:query]) - end - - unless params[:search].blank? - logger.debug "Searching for #{params[:search]}" - @packages = @packages.general_search(params[:search]) - end - - # Limit the packages to those that have approved screenshots. - # Also eager-load the screenshots. - if params[:show]=='onlywith' - @packages = @packages.where("screenshots.approved"=>true) - end - - if params[:show]=='with' - @packages = @packages.with_screenshots - logger.debug 'Limiting packages to those with screenshots' - elsif params[:show]=='without' - @packages = @packages.without_screenshots - logger.debug 'Limiting packages to those without screenshots' - end - end - def list - index + @packages = query_packages @packages = @packages.paginate(page: params[:page], per_page: 6) render 'packages/index-list.slim' end def grid - index + @packages = query_packages @packages = @packages.paginate(page: params[:page], per_page: 24) render 'packages/index-grid.slim' end @@ -93,10 +66,33 @@ class PackagesController < ApplicationController private # Send a dummy thumbnail reading "No screenshot available. Sorry." + # TODO: Make sure it uses X-Sendfile correctly in production def thumbnail404 send_file Rails.root.join('public/images/dummy/no-screenshots-available.png'), type: "image/png", disposition: 'inline', status: 404 end + + # Return packages matching the criteria given by parameters + def query_packages + packages = Package.includes(:screenshots) + + # text search + if params[:search].present? + logger.debug "Searching for #{params[:search]}" + packages = packages.general_search(params[:search]) + end + + case params[:show] + when 'with' + packages = packages.with_screenshots + logger.debug 'Limiting packages to those with screenshots' + when 'without' + packages = packages.without_screenshots + logger.debug 'Limiting packages to those without screenshots' + end + + return packages + end end diff --git a/app/views/packages/details.slim b/app/views/packages/details.slim index 3fb3b15..fc4e061 100644 --- a/app/views/packages/details.slim +++ b/app/views/packages/details.slim @@ -4,20 +4,20 @@ .small-7.medium-7.columns p.subtitle = @package.description - - - @package.screenshots.each do |screenshot| - .row.listview - .small-12.columns - a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption - = image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption) - .imgcaption =screenshot.caption - .imgstatus =screenshot.status - // TODO: Enable button only if admin or uploader - a.button.tiny.alert[ - href=delete_screenshot_path(@package.name, screenshot.id) - onclick="return confirm('Really delete the screenshot?');" - ] Delete screenshot - + - if @package.screenshots.count > 0 + - @package.screenshots.each do |screenshot| + .row.listview + .small-12.columns + a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption + = image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption) + .imgcaption =screenshot.caption + // TODO: Enable button only if admin or uploader + a.button.tiny.alert[ + href=delete_screenshot_path(@package.name, screenshot.id) + onclick="return confirm('Really delete the screenshot?');" + ] Delete screenshot + - else + img src='/images/dummy/no-screenshots-upload-one.svg' width='100%' // TODO: Enable comments in a later version // = partial '/package/comments' @@ -26,15 +26,9 @@ .bigpanel // Use action long description // TODO: Use actual description instead of placeholder text - p - ' Mail client suitable for free distribution. It supports different mail - ' accounts (POP, IMAP, Gmail), has an integrated learning Spam filter, and - ' offers easy organization of mails with tagging and virtual folders. Also, - ' more features can be added by installing extensions. - - p - ' The goal of Icedove is to produce a cross platform standalone mail - ' application using the XUL user interface language. + // TODO: split long_description by

tags + // TODO: What to do if long description is missing? + p =@package.long_description a.button.round.expand data-reveal-id="upload-modal" Upload more screenshots for #{@package.name} diff --git a/app/views/welcome/about.slim b/app/views/welcome/about.slim index c522fb3..db7df42 100644 --- a/app/views/welcome/about.slim +++ b/app/views/welcome/about.slim @@ -86,6 +86,16 @@ ' If you would like to get a complete list of packages or screenshots in JSON ' format then contact us and we will give you the URL. + h2 Some packages are not found + + p + ' That's right. We import the package repositories from various Linux + ' distributions once a day. However we already filter out packages that + ' we think would not be able to have a meaningful screenshot. + ' Like libraries or documentation or developer packages. So it may happen + ' that you a package is not found on this web sites although it may be + ' available in your favorite Linux distribution. + h2 Uploading new screenshots p diff --git a/app/views/welcome/home.slim b/app/views/welcome/home.slim index a08f186..5345053 100644 --- a/app/views/welcome/home.slim +++ b/app/views/welcome/home.slim @@ -25,37 +25,37 @@ div.row hr -div.row - div.large-4.medium-12.small-12.columns.text-center - h2 Newest upload - // TODO: load actual newest upload and cache it - // TODO: link to the package page - a.black href=package_path(@newest_uploaded_package.name) - img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false) - p - strong - =@newest_uploaded_package.name - ' : - ' - =@newest_uploaded_package.description - div.large-4.medium-12.small-12.columns.text-center - h2 Most popular - // TODO: load actual most popular package and highest-rated screenshot and cache it - // TODO: link to the package page - a.black href=package_path(@most_popular_package.name) - img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false) - p - strong - =@most_popular_package.name - ' : - ' - =@most_popular_package.description - div.large-4.medium-12.small-12.columns.text-center - h2 Your help needed - // TODO: load actual most popular package without a screenshot and cache it - a href=upload_package_by_name_path('firefox') - img src="/images/dummy/no-screenshots-upload-one.svg" width="80%" - p - ' We lack a screenshot for the Firefox package. Can you - a href=upload_package_by_name_path('firefox') provide one - ' ? +/ div.row +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Newest upload +/ // TODO: load actual newest upload and cache it +/ // TODO: link to the package page +/ a.black href=package_path(@newest_uploaded_package.name) +/ img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false) +/ p +/ strong +/ =@newest_uploaded_package.name +/ ' : +/ ' +/ =@newest_uploaded_package.description +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Most popular +/ // TODO: load actual most popular package and highest-rated screenshot and cache it +/ // TODO: link to the package page +/ a.black href=package_path(@most_popular_package.name) +/ img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false) +/ p +/ strong +/ =@most_popular_package.name +/ ' : +/ ' +/ =@most_popular_package.description +/ div.large-4.medium-12.small-12.columns.text-center +/ h2 Your help needed +/ // TODO: load actual most popular package without a screenshot and cache it +/ a href=upload_package_by_name_path('firefox') +/ img src="/images/dummy/no-screenshots-upload-one.svg" width="80%" +/ p +/ ' We lack a screenshot for the Firefox package. Can you +/ a href=upload_package_by_name_path('firefox') provide one +/ ' ? diff --git a/bin/rails b/bin/rails index 728cd85..7feb6a3 100755 --- a/bin/rails +++ b/bin/rails @@ -1,4 +1,8 @@ #!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end APP_PATH = File.expand_path('../../config/application', __FILE__) require_relative '../config/boot' require 'rails/commands' diff --git a/bin/rake b/bin/rake index 1724048..8017a02 100755 --- a/bin/rake +++ b/bin/rake @@ -1,4 +1,8 @@ #!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/bin/rspec b/bin/rspec new file mode 100755 index 0000000..20060eb --- /dev/null +++ b/bin/rspec @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby +begin + load File.expand_path("../spring", __FILE__) +rescue LoadError +end +require 'bundler/setup' +load Gem.bin_path('rspec-core', 'rspec') diff --git a/bin/spring b/bin/spring new file mode 100755 index 0000000..7b45d37 --- /dev/null +++ b/bin/spring @@ -0,0 +1,15 @@ +#!/usr/bin/env ruby + +# This file loads spring without using Bundler, in order to be fast. +# It gets overwritten when you run the `spring binstub` command. + +unless defined?(Spring) + require "rubygems" + require "bundler" + + if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m) + Gem.paths = { "GEM_PATH" => [Bundler.bundle_path.to_s, *Gem.path].uniq } + gem "spring", match[1] + require "spring/binstub" + end +end diff --git a/config/database.yml b/config/database.yml index 187ad98..1e0e165 100644 --- a/config/database.yml +++ b/config/database.yml @@ -7,10 +7,10 @@ development: #adapter: sqlite3 #database: db/development.sqlite3 adapter: postgresql - database: debshots + database: debshots_dev username: debshots - password: shootme - host: torf + password: GonwannEn0 + host: localhost pool: 5 timeout: 5000 @@ -19,10 +19,10 @@ development: # Do not set this db to the same as development or production. test: adapter: postgresql - database: debshots + database: debshots_test username: debshots - password: shootme - host: torf + password: GonwannEn0 + host: localhost pool: 5 timeout: 5000 diff --git a/config/environments/development.rb b/config/environments/development.rb index b39331a..ced472c 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -34,10 +34,11 @@ Debshots::Application.configure do # URL prefix leading to the static images that should get delivered by the web server config.images_path_prefix = '/screenshots' - # DEB package repositories to parse when running "rake import_debian" + # DEB package repositories to parse when running "rake debshots:... tasks" config.package_sources = [ { description: 'Development test files', type: 'apt', url: 'lib/tasks/files/', + architectures: ['i386', 'amd64'] #components: ['main','restricted','universe','multiverse'] } ] diff --git a/config/environments/production.rb b/config/environments/production.rb index 0bdc398..a5f1794 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -10,6 +10,9 @@ Debshots::Application.configure do # Rake tasks automatically ignore this option for performance. config.eager_load = true + # ActiveRecord errors propagate normally (forward deprecation to Rails 5) + config.active_record.raise_in_transactional_callbacks = true + # Full error reports are disabled and caching is turned on. config.consider_all_requests_local = false config.action_controller.perform_caching = true @@ -20,7 +23,6 @@ Debshots::Application.configure do # config.action_dispatch.rack_cache = true # Disable Rails's static asset server (Apache or nginx will already do this). - # config.serve_static_assets = false config.serve_static_files = true # Compress JavaScripts and CSS. @@ -87,6 +89,7 @@ Debshots::Application.configure do config.package_sources = [ { description: 'Debian Unstable (Sid)', type: 'apt', url: 'http://ftp.de.debian.org/debian/dists/sid', + architectures: ['i386' ,'amd64'] #components: ['main','restricted','universe','multiverse'] } ] diff --git a/config/environments/test.rb b/config/environments/test.rb index bf7a212..4f154be 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -12,8 +12,14 @@ Debshots::Application.configure do # preloads Rails for running tests, you may have to set it to true. config.eager_load = false + # ActiveRecord errors propagate normally (forward deprecation to Rails 5) + config.active_record.raise_in_transactional_callbacks = true + + # Test :sorted or :random. ":random" may be safer but harder to debug. + config.active_support.test_order = :sorted + # Configure static asset server for tests with Cache-Control for performance. - config.serve_static_assets = true + config.serve_static_files = true config.static_cache_control = "public, max-age=3600" # Show full error reports and disable caching. @@ -36,4 +42,20 @@ Debshots::Application.configure do # URL prefix leading to the static images that should get delivered by the web server config.images_path_prefix = '/screenshots' + + # DEB package repositories to parse when running "rake debshots:... tasks" + config.package_sources = [ + { + description: 'Development test files', type: 'apt', url: 'lib/tasks/files/', + architectures: ['amd64'], + components: ['main'] + } + ] + + config.images_path = Rails.root.join('public', 'screenshots') + + config.image_sizes = { + large: '800x600', + small: '160x120', + } end diff --git a/db/migrate/20150422223156_change_package_longdescription_to_text.rb b/db/migrate/20150422223156_change_package_longdescription_to_text.rb new file mode 100644 index 0000000..80ac3a4 --- /dev/null +++ b/db/migrate/20150422223156_change_package_longdescription_to_text.rb @@ -0,0 +1,5 @@ +class ChangePackageLongdescriptionToText < ActiveRecord::Migration + def change + change_column :packages, :long_description, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 489ff6c..61b2333 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20150103194230) do +ActiveRecord::Schema.define(version: 20150422223156) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -31,7 +31,7 @@ ActiveRecord::Schema.define(version: 20150103194230) do t.string "maintainer_email", limit: 100 t.string "homepage", limit: 400 t.string "version", limit: 200 - t.string "long_description", limit: 2000 + t.text "long_description" t.string "origin", limit: 80 t.datetime "created_at" t.datetime "updated_at" diff --git a/lib/deb_importer.rb b/lib/deb_importer.rb index 05d3e89..b85dba4 100644 --- a/lib/deb_importer.rb +++ b/lib/deb_importer.rb @@ -1,5 +1,7 @@ # Various helper methods to update the database of packages +require 'bzip2' + module DebImporter # This module imports information about packages of a Linux distribution @@ -15,9 +17,9 @@ module DebImporter attr_reader :architectures, :components, :description, :codename, :origin, :version, :files # Load and parse a Release file of an APT repository - def initialize(dist_url, components) + def initialize(dist_url) @dist_url = dist_url - release_url = dist_url + "Release" + release_url = dist_url + "/Release" Rails.logger.debug "Loading Release file from #{release_url}" open(release_url) do |release_data| fields = get_fields(release_data) @@ -27,51 +29,72 @@ module DebImporter @codename = fields[:Codename] @origin = fields[:Origin] @version = fields[:Version] - # Not needed at the moment. We do not check sizes or checksums yet. - #when 'SHA1', 'SHA256', 'MD5Sum' - # next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them - # @files = [] - # value.lines.each do |line| - # path = line.split.last - # @files << path - # end + + # TODO: check sizes and checksums end # open end # def initialize + # Get package information from translation (i18n) files. + # Returns an enumerator of packages. + def i18n(component, language) + url = "#{@dist_url}/#{component}/i18n/Translation-en" + file = find_and_open_compressed_url(url) + if file + return get_paragraphs(file) + else + return [] + end + end + + # Look for the file or URL in various compressed formats + # (e.g. bz2, gz) and fall back to plain text format. + def find_and_open_compressed_url(base_url) + Rails.logger.debug "Looking for files at URL #{base_url} with different compressions" + for suffix in ['.bz2', '.gz', ''] + begin + begin + url = "#{base_url}#{suffix}" + Rails.logger.debug "Checking if file at #{url} is available" + file = open(url) + rescue OpenURI::HTTPError => e + Rails.logger.debug "Loading #{url} lead to error #{e}. skipping." + next + end + + # Decompress file depending on its filename suffix + case suffix + when '.bz2' + file = Bzip2::Reader.new(file) + when '.gz' + file = Zlib::GzipReader.new(file) + end + Rails.logger.debug "File containing translations is: #{file}" + + return file + rescue Errno::ENOENT + Rails.logger.debug "URL #{url} could not be opened. Skipping." + end + end + + Rails.logger.error "No file found at #{url} and various compression extensions." + return nil + end + # Try to load the Packages file for a certain component (e.g. "main") # and architecture (e.g. "amd64") def packages(component, architecture) # create path like "main/binary-amd64/Packages" - packages_path = "#{component}/binary-#{architecture}/Packages" - # Check if gzip or uncompressed files exist - # (there is no working bzip2 library for Ruby 2.x at the time - 11/2014) - for suffix in ['.gz', ''] - packages_path_with_suffix = packages_path+suffix - Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}" - url = @dist_url + packages_path_with_suffix - begin - Rails.logger.debug "Try opening URL: #{url}" - file = open(url) - if suffix == '.gz' - file = Zlib::GzipReader.new(file) - end - Rails.logger.debug "File containing packages is: #{file}" - #paragraphs = get_paragraphs(file) - #return paragraphs - #byebug - return get_paragraphs(file) - rescue Errno::ENOENT - Rails.logger.debug "URL could not be opened. Skipping." - end - end - return nil + packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages" + Rails.logger.debug "Loading packages from #{packages_path}" + file = find_and_open_compressed_url(packages_path) + return get_paragraphs(file) end # def packages end # class Release private - # Gather the fields of a Debian control file and return them as a hash + # Gather the fields of a Debian control file section and return them as a hash def get_fields(data) fields = {} @@ -81,12 +104,16 @@ module DebImporter when /^(.+): (.+)/ # "Key: Value" fields[name.to_sym]=value unless value.empty? name,value=$1,$2 - when /^(.+):\s*/ # "Key:" (start of multi-line entry) + when /^(.+):$/ # "Key:" (start of multi-line entry without value in line) fields[name.to_sym]=value unless value.empty? name=$1 value='' when /^\s+(.+)/ # " Indented multi-line value" - value << $1+"\n" + # Add a newline for multi-line entries ("Key: Value\n Foo\n Bar") + unless value.empty? + value << "\n" + end + value << $1 when /^\s+$/ # Empty line break end diff --git a/lib/tasks/import_debian.rake b/lib/tasks/import_debian.rake index dfd6c63..946eaf5 100644 --- a/lib/tasks/import_debian.rake +++ b/lib/tasks/import_debian.rake @@ -36,75 +36,188 @@ BLACKLIST_SECTION_PATTERN = [ /\/?cli-mono$/, ] -# Whether to delete a packag +# Whether to delete a blacklisted package from the database REMOVE_BLACKLISTED_PACKAGE = true -desc "Import package information from Debian-style Packages files" +namespace :debshots do + desc "Import/update package database from configured DEB repositories" -task :import_debian => :environment do - repositories = Rails.configuration.package_sources - Rails.logger = Logger.new(STDOUT) - Rails.logger.level = Logger::INFO - #Rails.logger.level = Logger::DEBUG + task :update_from_deb_repos => :environment do + # Counters for added, updated or removed packages from the database + stats_added = 0 + stats_updated = 0 + stats_removed = 0 - Rails.logger.info "Importing Debian package information" + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + #Rails.logger.level = Logger::DEBUG - repositories.each do |repository| - Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" - release = DebImporter::Release.new(repository[:url], repository[:components]) - Rails.logger.info "> Supported architectures are: #{release.architectures}" - Rails.logger.info "> Supported components are: #{release.components}" + Rails.logger.info "Importing Debian package information" - # TODO: Remember what we imported to show it on the /about page + repositories.each do |repository| + wanted_architectures = repository[:architectures] + Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" + release = DebImporter::Release.new(repository[:url]) + Rails.logger.info "> Supported architectures are: #{release.architectures}" + if wanted_architectures + Rails.logger.info "> We only want architectures: #{wanted_architectures}" + end + Rails.logger.info "> Supported components are: #{release.components}" - release.components.split.each do |component| - Rails.logger.info "> Component: #{component}" - release.architectures.split.each do |architecture| - Rails.logger.info ">> Architecture: #{architecture}" - # Check if this component and architecture is available on this mirror - packages = release.packages(component, architecture) - unless packages - Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" - next - end - - Rails.logger.info "Packages file for #{component} on #{architecture} found." - Rails.logger.debug "Got: #{packages}" - packages.each do |package| - Rails.logger.info "> Package: #{package[:Package]}" - #Rails.logger.debug "Fetching package informaton from the database" - - db_package = Package.find_by name: package[:Package] - - if package_blacklisted? package - # Should the package get removed from the database? - if REMOVE_BLACKLISTED_PACKAGE - if db_package - Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" - db_package.destroy - end - end + # TODO: Remember what we imported to show it on the /about page + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.architectures.split.each do |architecture| + if wanted_architectures and not wanted_architectures.include?(architecture) + Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." next - end # if blacklisted - - unless db_package - Rails.logger.debug "No such package in our database." + end + Rails.logger.info ">> Architecture: #{architecture}" + # Check if this component and architecture is available on this mirror + packages = release.packages(component, architecture) + unless packages + Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror" next end - Rails.logger.debug "Found in database: #{db_package.id}" - update_data(package, db_package) - end # package.each + Rails.logger.info "Packages file for #{component} on #{architecture} found." + Rails.logger.debug "Got: #{packages}" + packages.each do |package| + Rails.logger.info "> Package: #{package[:Package]}" + #Rails.logger.debug "Fetching package informaton from the database" - end # architectures.each - end # components.each + if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section] + # Should the package get removed from the database? + if REMOVE_BLACKLISTED_PACKAGE + db_package = Package.find_by name: package[:Package] + if db_package + Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database" + db_package.destroy + stats_removed +=1 + end + end - Rails.logger.info "Done parsing #{repository[:url]} repository." - Rails.logger.info "--------------------------------------------" - end # repositories.each + next + end # if blacklisted -end # task + db_package = Package.find_by name: package[:Package] + unless db_package + Rails.logger.debug "No such package in our database. Creating one." + db_package = Package.create + stats_added += 1 + end + + # Rails.logger.debug "Found in database: #{db_package}" + update_data(package, db_package) + stats_updated += 1 + end # package.each + + end # architectures.each + end # components.each + + Rails.logger.info "Done parsing #{repository[:url]} repository." + Rails.logger.info "#{stats_added} packages added" + Rails.logger.info "#{stats_updated} packages updated" + Rails.logger.info "#{stats_removed} packages removed" + Rails.logger.info "--------------------------------------------" + end # repositories.each + + end # task + + desc 'Update long description from i18n file' + + task :update_longdescription_from_deb_repos => :environment do + + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + Rails.logger.info "Importing long descriptoin from Debian repository (i18n)" + + repositories.each do |repository| + Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}" + release = DebImporter::Release.new(repository[:url]) + Rails.logger.info "> Supported components are: #{release.components}" + + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.i18n(component, 'en').each do |pkg| + Rails.logger.debug "i18n information: #{pkg}" + unless pkg[:'Description-en'] + Rails.logger.debug "No Description-en found in section." + next + end + if package_name_blacklisted? pkg[:Package] + next + end + # See if we have that package in the database + if db_pkg = Package.find_by(name: pkg[:Package]) + Rails.logger.info "Updating long description for package #{db_pkg.name}" + text = pkg[:'Description-en'] + short_description, long_description = text.split("\n", 2) + db_pkg.long_description = long_description + db_pkg.save + else + Rails.logger.debug "Package not in database. Nothing to update." + end + end # pkg.each + end # components.each + + + Rails.logger.info "Done updating long descriptions from translation file." + Rails.logger.info "--------------------------------------------" + end # repositories.each + + end # task + + # TODO: Some code duplication. Should be refactord. + desc "List configured DEB repositories" + + task :list_deb_repos => :environment do + repositories = Rails.configuration.package_sources + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + Rails.logger.info "Listing configured DEB repositories" + + repositories.each do |repository| + wanted_architectures = repository[:architectures] + Rails.logger.info "Fetching Release file for repository: #{repository[:url]}" + release = DebImporter::Release.new(repository[:url]) + Rails.logger.info "> Supported architectures are: #{release.architectures}" + if wanted_architectures + Rails.logger.info "> We only want architectures: #{wanted_architectures}" + end + Rails.logger.info "> Supported components are: #{release.components}" + + release.components.split.each do |component| + Rails.logger.info "> Component: #{component}" + release.architectures.split.each do |architecture| + if wanted_architectures and not wanted_architectures.include?(architecture) + Rails.logger.debug "Architecture #{architecture} not wanted. Skipping." + next + end + + Rails.logger.info ">> Architecture: #{architecture}" + # Check if this component and architecture is available on this mirror + packages = release.packages(component, architecture) + if packages + Rails.logger.info "#{packages.count} packages found." + else + Rails.logger.info "No packages." + end + end # architectures.each + end # components.each + + end # repositories.each + + end # task + +end # namespace # Update the information about a package in the database def update_data(package, db_package) @@ -126,33 +239,26 @@ def update_data(package, db_package) end db_package.save - - # TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en end -# Check if a package is blacklisted and should not be imported. +# Check if a package name is blacklisted and should not be imported. # This prevents importing boring software that likely does not get a useful screenshot. -# Argument: Hash containing package information from a Packages file -def package_blacklisted?(package) - Rails.logger.debug "Blacklist check of package #{package[:Package]}" - - # TODO: add a flag that allows to manually blacklist a package - - BLACKLIST_SECTION_PATTERN.each do |pattern| - Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}" - if package[:Section] =~ pattern - Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')" - return true - end +def package_name_blacklisted?(name) + if BLACKLIST_NAME_PATTERN.find { |pattern| name=~pattern} + Rails.logger.debug " > Blacklisted by name ('#{name}')" + return true + else + return false + end +end + +# Check if a package's section is blacklisted and should not be imported. +# This prevents importing boring software that likely does not get a useful screenshot. +def package_section_blacklisted?(section) + if BLACKLIST_SECTION_PATTERN.find { |pattern| section=~pattern} + Rails.logger.debug " > Blacklisted by section ('#{section}')" + return true + else + return false end - - BLACKLIST_NAME_PATTERN.each do |pattern| - Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}" - if package[:Package] =~ pattern - Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')" - return true - end - end - - false end diff --git a/test/controllers/packages_controller_test.rb b/test/controllers/packages_controller_test.rb index f78cc4e..b3bcc38 100644 --- a/test/controllers/packages_controller_test.rb +++ b/test/controllers/packages_controller_test.rb @@ -2,23 +2,23 @@ require 'test_helper' class PackagesControllerTest < ActionController::TestCase test "should get index" do - get :index + get :grid assert_response :success end - test "should get with_screenshots" do - get :with_screenshots - assert_response :success - end + # test "should get with_screenshots" do + # get :with_screenshots + # assert_response :success + # end - test "should get without_screenshots" do - get :without_screenshots - assert_response :success - end + # test "should get without_screenshots" do + # get :without_screenshots + # assert_response :success + # end - test "should get moderate" do - get :moderate - assert_response :success - end + # test "should get moderate" do + # get :moderate + # assert_response :success + # end end diff --git a/test/controllers/welcome_controller_test.rb b/test/controllers/welcome_controller_test.rb index dff8e9d..23d1cc7 100644 --- a/test/controllers/welcome_controller_test.rb +++ b/test/controllers/welcome_controller_test.rb @@ -2,8 +2,7 @@ require 'test_helper' class WelcomeControllerTest < ActionController::TestCase test "should get index" do - get :index + get :home assert_response :success end - end diff --git a/test/fixtures/packages.yml b/test/fixtures/packages.yml index 0ac2d56..3b82496 100644 --- a/test/fixtures/packages.yml +++ b/test/fixtures/packages.yml @@ -1,23 +1,23 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html -one: - name: MyText - description: MyString - long_description: MyString - section: MyString - maintainer: MyString - maintainer_email: MyString - homepage: MyString - version: MyString - origin: MyString +# one: +# name: MyText +# description: MyString +# long_description: MyString +# section: MyString +# maintainer: MyString +# maintainer_email: MyString +# homepage: MyString +# version: MyString +# origin: MyString -two: - name: MyText - description: MyString - long_description: MyString - section: MyString - maintainer: MyString - maintainer_email: MyString - homepage: MyString - version: MyString - origin: MyString +# two: +# name: MyText +# description: MyString +# long_description: MyString +# section: MyString +# maintainer: MyString +# maintainer_email: MyString +# homepage: MyString +# version: MyString +# origin: MyString diff --git a/test/fixtures/screenshots.yml b/test/fixtures/screenshots.yml index 5ace412..b6e965a 100644 --- a/test/fixtures/screenshots.yml +++ b/test/fixtures/screenshots.yml @@ -1,21 +1,21 @@ # Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html -one: - version: MyString - uploaddatetime: 2013-07-27 13:29:55 - uploaderhash: MyString - uploaderip: MyString - approved: - markedfordelete: - delete_reason: MyString - description: MyString +# one: +# version: MyString +# uploaddatetime: 2013-07-27 13:29:55 +# uploaderhash: MyString +# uploaderip: MyString +# approved: +# markedfordelete: +# delete_reason: MyString +# description: MyString -two: - version: MyString - uploaddatetime: 2013-07-27 13:29:55 - uploaderhash: MyString - uploaderip: MyString - approved: - markedfordelete: - delete_reason: MyString - description: MyString +# two: +# version: MyString +# uploaddatetime: 2013-07-27 13:29:55 +# uploaderhash: MyString +# uploaderip: MyString +# approved: +# markedfordelete: +# delete_reason: MyString +# description: MyString diff --git a/test/lib/deb_importer_test.rb b/test/lib/deb_importer_test.rb new file mode 100644 index 0000000..cee922c --- /dev/null +++ b/test/lib/deb_importer_test.rb @@ -0,0 +1,40 @@ +$LOAD_PATH << 'lib' + +require 'open-uri' # allows to load URLs using open() +require 'deb_importer' +require 'test_helper' + +include DebImporter + +class PackagesHelperTest < ActionView::TestCase + + test "should be able to parse local Debian repository test files" do + Rails.logger = Logger.new(STDOUT) + Rails.logger.level = Logger::ERROR + # Rails.logger.level = Logger::INFO + # Rails.logger.level = Logger::DEBUG + + # Load information about test repository from environments/test.rb + repositories = Rails.configuration.package_sources + + Rails.logger.info "Test parsing local DEB-style repository" + + repository = repositories.first + release = DebImporter::Release.new(repository[:url]) + assert_equal release.components, 'main restricted universe multiverse' + + # Packages enumerator must be restarted to start from position 1 + packages = release.packages('main', 'amd64') + assert_instance_of Enumerator, packages + first_package = packages.first + assert_equal 'account-plugin-aim', first_package[:Package] + + # Packages enumerator must be restarted to start from position 1 + packages = release.packages('main', 'amd64') + assert_equal 8320, packages.count + + # Get information from translation file + i18n = release.i18n('main', 'en') + assert_match /^Real-time strategy game of ancient warfare\n0 A\.D\./, i18n.first[:'Description-en'] + end # test +end # class diff --git a/test/test_helper.rb b/test/test_helper.rb index bc7e05d..6e9cc65 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,15 +1,17 @@ -ENV["RAILS_ENV"] ||= "test" -require File.expand_path('../../config/environment', __FILE__) -require 'rails/test_help' +ENV['RAILS_ENV'] ||= 'test' +require File.expand_path("../../config/environment", __FILE__) +require "rails/test_help" +# require "minitest/rails" +# require "minitest/autorun" + +# To add Capybara feature tests add `gem "minitest-rails-capybara"` +# to the test group in the Gemfile and uncomment the following: +# require "minitest/rails/capybara" + +# Uncomment for awesome colorful output +# require "minitest/pride" class ActiveSupport::TestCase - ActiveRecord::Migration.check_pending! - - # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order. - # - # Note: You'll currently still have to declare fixtures explicitly in integration tests - # -- they do not yet inherit this setting - fixtures :all - - # Add more helper methods to be used by all tests here... + # Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order. + fixtures :all end