Merge branch 'master' of bitbucket.org:signum/debshots

Conflicts:
	app/views/packages/details.slim
This commit is contained in:
Christoph Haas 2015-04-25 20:09:28 +02:00
commit ce5f99887d
28 changed files with 656 additions and 352 deletions

View file

@ -67,6 +67,12 @@ group :development, :test do
# Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
gem 'spring'
# Helps run tests automatically after changes
# TODO: not sure all these packages are required (http://buildingrails.com/a/rails_automated_testing_setup_for_beginners)
gem 'guard-rails'
gem 'minitest-rails'
gem 'guard-minitest'
end
# Zurb Foundation
@ -93,3 +99,6 @@ gem "paperclip", "~> 4.2"
# Comply with stupid european cookie law
gem 'cookies_eu'
# Requires: apt-get install libbz2-dev
gem 'bzip2-ruby', :git => 'https://github.com/chewi/bzip2-ruby.git'

View file

@ -1,3 +1,9 @@
GIT
remote: https://github.com/chewi/bzip2-ruby.git
revision: 92d492c2b40c32a5c039314666655ebc2f52a817
specs:
bzip2-ruby (0.2.7)
GEM
remote: https://rubygems.org/
specs:
@ -46,6 +52,8 @@ GEM
builder (3.2.2)
byebug (4.0.5)
columnize (= 0.9.0)
celluloid (0.16.0)
timers (~> 4.0.0)
choice (0.1.7)
climate_control (0.0.3)
activesupport (>= 3.0)
@ -68,11 +76,30 @@ GEM
execjs (2.5.2)
fancybox2-rails (0.2.8)
railties (>= 3.1.0, < 5.0)
ffi (1.9.8)
formatador (0.2.5)
foundation-rails (5.5.1.2)
railties (>= 3.1.0)
sass (>= 3.3.0, < 3.5)
globalid (0.3.5)
activesupport (>= 4.1.0)
guard (2.12.5)
formatador (>= 0.2.4)
listen (~> 2.7)
lumberjack (~> 1.0)
nenv (~> 0.1)
notiffany (~> 0.0)
pry (>= 0.9.12)
shellany (~> 0.0)
thor (>= 0.18.1)
guard-compat (1.2.1)
guard-minitest (2.4.4)
guard-compat (~> 1.2)
minitest (>= 3.0)
guard-rails (0.7.1)
guard (~> 2.11)
guard-compat (~> 1.0)
hitimes (1.2.2)
i18n (0.7.0)
jbuilder (2.2.13)
activesupport (>= 3.0.0, < 5)
@ -84,16 +111,29 @@ GEM
railties (>= 4.2.0)
thor (>= 0.14, < 2.0)
json (1.8.2)
listen (2.10.0)
celluloid (~> 0.16.0)
rb-fsevent (>= 0.9.3)
rb-inotify (>= 0.9)
loofah (2.0.1)
nokogiri (>= 1.5.9)
lumberjack (1.0.9)
mail (2.6.3)
mime-types (>= 1.16, < 3)
method_source (0.8.2)
mime-types (2.4.3)
mini_portile (0.6.2)
minitest (5.6.0)
minitest-rails (2.1.1)
minitest (~> 5.4)
railties (~> 4.1)
multi_json (1.11.0)
nenv (0.2.0)
nokogiri (1.6.6.2)
mini_portile (~> 0.6.0)
notiffany (0.0.6)
nenv (~> 0.1)
shellany (~> 0.0)
paperclip (4.2.1)
activemodel (>= 3.0.0)
activesupport (>= 3.0.0)
@ -104,6 +144,10 @@ GEM
activerecord (>= 3.1)
activesupport (>= 3.1)
arel
pry (0.10.1)
coderay (~> 1.1.0)
method_source (~> 0.8.1)
slop (~> 3.4)
rack (1.6.0)
rack-test (0.6.3)
rack (>= 1.0)
@ -137,6 +181,9 @@ GEM
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
rake (10.4.2)
rb-fsevent (0.9.4)
rb-inotify (0.9.5)
ffi (>= 0.5.0)
rdoc (4.2.0)
json (~> 1.4)
ruby-graphviz (1.0.9)
@ -150,6 +197,7 @@ GEM
sdoc (0.4.1)
json (~> 1.7, >= 1.7.7)
rdoc (~> 4.0)
shellany (0.0.1)
slim (3.0.3)
temple (~> 0.7.3)
tilt (>= 1.3.3, < 2.1)
@ -159,6 +207,7 @@ GEM
activesupport (>= 3.1, < 5.0)
railties (>= 3.1, < 5.0)
slim (~> 3.0)
slop (3.6.0)
spring (1.3.4)
sprockets (3.0.1)
rack (~> 1.0)
@ -170,6 +219,8 @@ GEM
thor (0.19.1)
thread_safe (0.3.5)
tilt (1.4.1)
timers (4.0.1)
hitimes
tzinfo (1.2.2)
thread_safe (~> 0.1)
uglifier (2.7.1)
@ -191,12 +242,16 @@ DEPENDENCIES
better_errors
binding_of_caller
byebug
bzip2-ruby!
coffee-rails (~> 4.1.0)
cookies_eu
fancybox2-rails (~> 0.2.8)
foundation-rails
guard-minitest
guard-rails
jbuilder (~> 2.0)
jquery-rails
minitest-rails
paperclip (~> 4.2)
pg
pg_search

55
Guardfile Normal file
View file

@ -0,0 +1,55 @@
# A sample Guardfile
# More info at https://github.com/guard/guard#readme
## Uncomment and set this to only include directories you want to watch
# directories %w(app lib config test spec features)
## Uncomment to clear the screen before every task
# clearing :on
## Guard internally checks for changes in the Guardfile and exits.
## If you want Guard to automatically start up again, run guard in a
## shell loop, e.g.:
##
## $ while bundle exec guard; do echo "Restarting Guard..."; done
##
## Note: if you are using the `directories` clause above and you are not
## watching the project directory ('.'), then you will want to move
## the Guardfile to a watched dir and symlink it back, e.g.
#
# $ mkdir config
# $ mv Guardfile config/
# $ ln -s config/Guardfile .
#
# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
# Guard-Rails supports a lot options with default values:
# daemon: false # runs the server as a daemon.
# debugger: false # enable ruby-debug gem.
# environment: 'development' # changes server environment.
# force_run: false # kills any process that's holding the listen port before attempting to (re)start Rails.
# pid_file: 'tmp/pids/[RAILS_ENV].pid' # specify your pid_file.
# host: 'localhost' # server hostname.
# port: 3000 # server port number.
# root: '/spec/dummy' # Rails' root path.
# server: thin # webserver engine.
# start_on_start: true # will start the server when starting Guard.
# timeout: 30 # waits untill restarting the Rails server, in seconds.
# zeus_plan: server # custom plan in zeus, only works with `zeus: true`.
# zeus: false # enables zeus gem.
# CLI: 'rails server' # customizes runner command. Omits all options except `pid_file`!
#guard 'rails' do
# watch('Gemfile.lock')
# watch(%r{^(config|lib)/.*})
#end
guard :minitest do
watch(%r{^app/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
watch(%r{^app/controllers/application_controller\.rb$}) { 'test/controllers' }
watch(%r{^app/controllers/(.+)_controller\.rb$}) { |m| "test/integration/#{m[1]}_test.rb" }
watch(%r{^app/views/(.+)_mailer/.+}) { |m| "test/mailers/#{m[1]}_mailer_test.rb" }
watch(%r{^lib/(.+)\.rb$}) { |m| "test/lib/#{m[1]}_test.rb" }
watch(%r{^test/.+_test\.rb$})
watch(%r{^test/test_helper\.rb$}) { 'test' }
end

View file

@ -34,4 +34,21 @@ Used: https://github.com/semaperepelitsa/jquery.fileupload-rails
Which is a gem for: https://github.com/blueimp/jQuery-File-Upload
Paperclip for attachment/screenshot file handlind within ActiveRecord
https://github.com/thoughtbot/paperclip
https://github.com/thoughtbot/paperclip
--
Components:
- Rails 5 (widespread, up-to-date, solves many common problems)
- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search)
- Git (most people nowadays use Git)
- jQuery (well known and easy to use)
- Zurb Foundation (integrates nicer than Twitter Bootstrap)
- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails)
--
Importer
How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat

View file

@ -107,76 +107,3 @@ TODO
/rss
* Return an RSS feed of new uploads of screenshots
== Using Xapian
Ruby gem "xapian" does not exist. Debian only has bindings for Ruby 1.9.x.
Xapian may be lightweight and nice but is too much hassle.
== Elasticsearch
= Install
Get it from https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-0.90.6.deb
Also: apt-get install openjdk-7-jre-headless
= Configure
Edit /etc/elasticsearch/elasticsearch.yml for single-server usage:
- cluster.name: ...
- node.name: ...
- index.number_of_shards: 1
- index.number_of_replicas: 0
- discovery.zen.ping.multicast.enabled: false
= Add useful plugins
/usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head
URL: serve:9200/_plugin/head
/usr/share/elasticsearch/bin/plugin -install lukas-vlcek/bigdesk
URL: serve:9200/_plugin/bigdesk
= Install River (that feeds PostgreSQL data into ElasticSearch):
Get the PostgreSQL ODBC driver from
wget http://jdbc.postgresql.org/download/postgresql-9.3-1100.jdbc41.jar
(according to http://jdbc.postgresql.org/download.html)
cp postgresql-9.3-1100.jdbc41.jar /usr/share/elasticsearch/plugins/river-jdbc
/usr/share/elasticsearch/bin/plugin -install river-jdbc -url http://bit.ly/1iovWV9
(according to https://github.com/jprante/elasticsearch-river-jdbc)
Fix permissions after plugin installation:
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch/plugins
Restart Elasticsearch.
curl -XPUT 'localhost:9200/_river/debshots_jdbc_river/_meta' -d '{
"type" : "jdbc",
"jdbc" : {
"driver" : "org.postgresql.Driver",
"url" : "jdbc:postgresql://localhost:5432/debshots",
"user" : "debshots",
"password" : "shootme",
"sql" : "select id as _id,name,description from packages",
"strategy" : "simple",
"poll" : "5m"
},
"index" : {
"index" : "debshots",
"type" : "jdbc"
}
}'
Should return:
{"ok":true,"_index":"_river","_type":"my_jdbc_river","_id":"_meta","_version":1}
TODO: Tire versus Rivers

View file

@ -1,7 +1,7 @@
// -------------- debshots ---------------------------
$page-background: #f2f2f2;
$page-background: #e8e8e8;
$footer-background: #333333;
$footer-text-color: #b3b3b3;
@ -148,6 +148,11 @@ html, body {
{
width: 100%;
}
img
{
box-shadow: 3px 3px 10px 0px #888888;
}
}
.row .gridview
@ -228,6 +233,7 @@ a.black
.imgcaption
{
text-align: center;
font-variant: italic;
}
/* Status of a screenshot (e.g. if it has to be moderated) */

View file

@ -1,39 +1,12 @@
class PackagesController < ApplicationController
def index
@packages = Package.includes(:screenshots)
if params[:query]
@packages = @packages.text_search(params[:query])
end
unless params[:search].blank?
logger.debug "Searching for #{params[:search]}"
@packages = @packages.general_search(params[:search])
end
# Limit the packages to those that have approved screenshots.
# Also eager-load the screenshots.
if params[:show]=='onlywith'
@packages = @packages.where("screenshots.approved"=>true)
end
if params[:show]=='with'
@packages = @packages.with_screenshots
logger.debug 'Limiting packages to those with screenshots'
elsif params[:show]=='without'
@packages = @packages.without_screenshots
logger.debug 'Limiting packages to those without screenshots'
end
end
def list
index
@packages = query_packages
@packages = @packages.paginate(page: params[:page], per_page: 6)
render 'packages/index-list.slim'
end
def grid
index
@packages = query_packages
@packages = @packages.paginate(page: params[:page], per_page: 24)
render 'packages/index-grid.slim'
end
@ -93,10 +66,33 @@ class PackagesController < ApplicationController
private
# Send a dummy thumbnail reading "No screenshot available. Sorry."
# TODO: Make sure it uses X-Sendfile correctly in production
def thumbnail404
send_file Rails.root.join('public/images/dummy/no-screenshots-available.png'),
type: "image/png",
disposition: 'inline',
status: 404
end
# Return packages matching the criteria given by parameters
def query_packages
packages = Package.includes(:screenshots)
# text search
if params[:search].present?
logger.debug "Searching for #{params[:search]}"
packages = packages.general_search(params[:search])
end
case params[:show]
when 'with'
packages = packages.with_screenshots
logger.debug 'Limiting packages to those with screenshots'
when 'without'
packages = packages.without_screenshots
logger.debug 'Limiting packages to those without screenshots'
end
return packages
end
end

View file

@ -4,20 +4,20 @@
.small-7.medium-7.columns
p.subtitle = @package.description
- @package.screenshots.each do |screenshot|
.row.listview
.small-12.columns
a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption
= image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption)
.imgcaption =screenshot.caption
.imgstatus =screenshot.status
// TODO: Enable button only if admin or uploader
a.button.tiny.alert[
href=delete_screenshot_path(@package.name, screenshot.id)
onclick="return confirm('Really delete the screenshot?');"
] Delete screenshot
- if @package.screenshots.count > 0
- @package.screenshots.each do |screenshot|
.row.listview
.small-12.columns
a.black.fancybox href=screenshot.image.url(:large, timestamp: false) rel='fancybox-thumb' title=screenshot.caption
= image_tag(screenshot.image.url(:large, timestamp: false), alt: screenshot.caption)
.imgcaption =screenshot.caption
// TODO: Enable button only if admin or uploader
a.button.tiny.alert[
href=delete_screenshot_path(@package.name, screenshot.id)
onclick="return confirm('Really delete the screenshot?');"
] Delete screenshot
- else
img src='/images/dummy/no-screenshots-upload-one.svg' width='100%'
// TODO: Enable comments in a later version
// = partial '/package/comments'
@ -26,15 +26,9 @@
.bigpanel
// Use action long description
// TODO: Use actual description instead of placeholder text
p
' Mail client suitable for free distribution. It supports different mail
' accounts (POP, IMAP, Gmail), has an integrated learning Spam filter, and
' offers easy organization of mails with tagging and virtual folders. Also,
' more features can be added by installing extensions.
p
' The goal of Icedove is to produce a cross platform standalone mail
' application using the XUL user interface language.
// TODO: split long_description by <p> tags
// TODO: What to do if long description is missing?
p =@package.long_description
a.button.round.expand data-reveal-id="upload-modal" Upload more screenshots for #{@package.name}

View file

@ -86,6 +86,16 @@
' If you would like to get a complete list of packages or screenshots in JSON
' format then contact us and we will give you the URL.
h2 Some packages are not found
p
' That's right. We import the package repositories from various Linux
' distributions once a day. However we already filter out packages that
' we think would not be able to have a meaningful screenshot.
' Like libraries or documentation or developer packages. So it may happen
' that you a package is not found on this web sites although it may be
' available in your favorite Linux distribution.
h2 Uploading new screenshots
p

View file

@ -25,37 +25,37 @@
div.row
hr
div.row
div.large-4.medium-12.small-12.columns.text-center
h2 Newest upload
// TODO: load actual newest upload and cache it
// TODO: link to the package page
a.black href=package_path(@newest_uploaded_package.name)
img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false)
p
strong
=@newest_uploaded_package.name
' :
'
=@newest_uploaded_package.description
div.large-4.medium-12.small-12.columns.text-center
h2 Most popular
// TODO: load actual most popular package and highest-rated screenshot and cache it
// TODO: link to the package page
a.black href=package_path(@most_popular_package.name)
img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false)
p
strong
=@most_popular_package.name
' :
'
=@most_popular_package.description
div.large-4.medium-12.small-12.columns.text-center
h2 Your help needed
// TODO: load actual most popular package without a screenshot and cache it
a href=upload_package_by_name_path('firefox')
img src="/images/dummy/no-screenshots-upload-one.svg" width="80%"
p
' We lack a screenshot for the Firefox package. Can you
a href=upload_package_by_name_path('firefox') provide one
' ?
/ div.row
/ div.large-4.medium-12.small-12.columns.text-center
/ h2 Newest upload
/ // TODO: load actual newest upload and cache it
/ // TODO: link to the package page
/ a.black href=package_path(@newest_uploaded_package.name)
/ img src=@newest_uploaded_package.screenshots.first.image.url(:large, timestamp: false)
/ p
/ strong
/ =@newest_uploaded_package.name
/ ' :
/ '
/ =@newest_uploaded_package.description
/ div.large-4.medium-12.small-12.columns.text-center
/ h2 Most popular
/ // TODO: load actual most popular package and highest-rated screenshot and cache it
/ // TODO: link to the package page
/ a.black href=package_path(@most_popular_package.name)
/ img src=@most_popular_package.screenshots.first.image.url(:large, timestamp: false)
/ p
/ strong
/ =@most_popular_package.name
/ ' :
/ '
/ =@most_popular_package.description
/ div.large-4.medium-12.small-12.columns.text-center
/ h2 Your help needed
/ // TODO: load actual most popular package without a screenshot and cache it
/ a href=upload_package_by_name_path('firefox')
/ img src="/images/dummy/no-screenshots-upload-one.svg" width="80%"
/ p
/ ' We lack a screenshot for the Firefox package. Can you
/ a href=upload_package_by_name_path('firefox') provide one
/ ' ?

View file

@ -1,4 +1,8 @@
#!/usr/bin/env ruby
begin
load File.expand_path("../spring", __FILE__)
rescue LoadError
end
APP_PATH = File.expand_path('../../config/application', __FILE__)
require_relative '../config/boot'
require 'rails/commands'

View file

@ -1,4 +1,8 @@
#!/usr/bin/env ruby
begin
load File.expand_path("../spring", __FILE__)
rescue LoadError
end
require_relative '../config/boot'
require 'rake'
Rake.application.run

7
bin/rspec Executable file
View file

@ -0,0 +1,7 @@
#!/usr/bin/env ruby
begin
load File.expand_path("../spring", __FILE__)
rescue LoadError
end
require 'bundler/setup'
load Gem.bin_path('rspec-core', 'rspec')

15
bin/spring Executable file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env ruby
# This file loads spring without using Bundler, in order to be fast.
# It gets overwritten when you run the `spring binstub` command.
unless defined?(Spring)
require "rubygems"
require "bundler"
if match = Bundler.default_lockfile.read.match(/^GEM$.*?^ (?: )*spring \((.*?)\)$.*?^$/m)
Gem.paths = { "GEM_PATH" => [Bundler.bundle_path.to_s, *Gem.path].uniq }
gem "spring", match[1]
require "spring/binstub"
end
end

View file

@ -7,10 +7,10 @@ development:
#adapter: sqlite3
#database: db/development.sqlite3
adapter: postgresql
database: debshots
database: debshots_dev
username: debshots
password: shootme
host: torf
password: GonwannEn0
host: localhost
pool: 5
timeout: 5000
@ -19,10 +19,10 @@ development:
# Do not set this db to the same as development or production.
test:
adapter: postgresql
database: debshots
database: debshots_test
username: debshots
password: shootme
host: torf
password: GonwannEn0
host: localhost
pool: 5
timeout: 5000

View file

@ -34,10 +34,11 @@ Debshots::Application.configure do
# URL prefix leading to the static images that should get delivered by the web server
config.images_path_prefix = '/screenshots'
# DEB package repositories to parse when running "rake import_debian"
# DEB package repositories to parse when running "rake debshots:... tasks"
config.package_sources = [
{
description: 'Development test files', type: 'apt', url: 'lib/tasks/files/',
architectures: ['i386', 'amd64']
#components: ['main','restricted','universe','multiverse']
}
]

View file

@ -10,6 +10,9 @@ Debshots::Application.configure do
# Rake tasks automatically ignore this option for performance.
config.eager_load = true
# ActiveRecord errors propagate normally (forward deprecation to Rails 5)
config.active_record.raise_in_transactional_callbacks = true
# Full error reports are disabled and caching is turned on.
config.consider_all_requests_local = false
config.action_controller.perform_caching = true
@ -20,7 +23,6 @@ Debshots::Application.configure do
# config.action_dispatch.rack_cache = true
# Disable Rails's static asset server (Apache or nginx will already do this).
# config.serve_static_assets = false
config.serve_static_files = true
# Compress JavaScripts and CSS.
@ -87,6 +89,7 @@ Debshots::Application.configure do
config.package_sources = [
{
description: 'Debian Unstable (Sid)', type: 'apt', url: 'http://ftp.de.debian.org/debian/dists/sid',
architectures: ['i386' ,'amd64']
#components: ['main','restricted','universe','multiverse']
}
]

View file

@ -12,8 +12,14 @@ Debshots::Application.configure do
# preloads Rails for running tests, you may have to set it to true.
config.eager_load = false
# ActiveRecord errors propagate normally (forward deprecation to Rails 5)
config.active_record.raise_in_transactional_callbacks = true
# Test :sorted or :random. ":random" may be safer but harder to debug.
config.active_support.test_order = :sorted
# Configure static asset server for tests with Cache-Control for performance.
config.serve_static_assets = true
config.serve_static_files = true
config.static_cache_control = "public, max-age=3600"
# Show full error reports and disable caching.
@ -36,4 +42,20 @@ Debshots::Application.configure do
# URL prefix leading to the static images that should get delivered by the web server
config.images_path_prefix = '/screenshots'
# DEB package repositories to parse when running "rake debshots:... tasks"
config.package_sources = [
{
description: 'Development test files', type: 'apt', url: 'lib/tasks/files/',
architectures: ['amd64'],
components: ['main']
}
]
config.images_path = Rails.root.join('public', 'screenshots')
config.image_sizes = {
large: '800x600',
small: '160x120',
}
end

View file

@ -0,0 +1,5 @@
class ChangePackageLongdescriptionToText < ActiveRecord::Migration
def change
change_column :packages, :long_description, :text
end
end

View file

@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20150103194230) do
ActiveRecord::Schema.define(version: 20150422223156) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -31,7 +31,7 @@ ActiveRecord::Schema.define(version: 20150103194230) do
t.string "maintainer_email", limit: 100
t.string "homepage", limit: 400
t.string "version", limit: 200
t.string "long_description", limit: 2000
t.text "long_description"
t.string "origin", limit: 80
t.datetime "created_at"
t.datetime "updated_at"

View file

@ -1,5 +1,7 @@
# Various helper methods to update the database of packages
require 'bzip2'
module DebImporter
# This module imports information about packages of a Linux distribution
@ -15,9 +17,9 @@ module DebImporter
attr_reader :architectures, :components, :description, :codename, :origin, :version, :files
# Load and parse a Release file of an APT repository
def initialize(dist_url, components)
def initialize(dist_url)
@dist_url = dist_url
release_url = dist_url + "Release"
release_url = dist_url + "/Release"
Rails.logger.debug "Loading Release file from #{release_url}"
open(release_url) do |release_data|
fields = get_fields(release_data)
@ -27,51 +29,72 @@ module DebImporter
@codename = fields[:Codename]
@origin = fields[:Origin]
@version = fields[:Version]
# Not needed at the moment. We do not check sizes or checksums yet.
#when 'SHA1', 'SHA256', 'MD5Sum'
# next if @files # skip parsing files if another section (e.g. "MD5Sum") already gathered them
# @files = []
# value.lines.each do |line|
# path = line.split.last
# @files << path
# end
# TODO: check sizes and checksums
end # open
end # def initialize
# Get package information from translation (i18n) files.
# Returns an enumerator of packages.
def i18n(component, language)
url = "#{@dist_url}/#{component}/i18n/Translation-en"
file = find_and_open_compressed_url(url)
if file
return get_paragraphs(file)
else
return []
end
end
# Look for the file or URL in various compressed formats
# (e.g. bz2, gz) and fall back to plain text format.
def find_and_open_compressed_url(base_url)
Rails.logger.debug "Looking for files at URL #{base_url} with different compressions"
for suffix in ['.bz2', '.gz', '']
begin
begin
url = "#{base_url}#{suffix}"
Rails.logger.debug "Checking if file at #{url} is available"
file = open(url)
rescue OpenURI::HTTPError => e
Rails.logger.debug "Loading #{url} lead to error #{e}. skipping."
next
end
# Decompress file depending on its filename suffix
case suffix
when '.bz2'
file = Bzip2::Reader.new(file)
when '.gz'
file = Zlib::GzipReader.new(file)
end
Rails.logger.debug "File containing translations is: #{file}"
return file
rescue Errno::ENOENT
Rails.logger.debug "URL #{url} could not be opened. Skipping."
end
end
Rails.logger.error "No file found at #{url} and various compression extensions."
return nil
end
# Try to load the Packages file for a certain component (e.g. "main")
# and architecture (e.g. "amd64")
def packages(component, architecture)
# create path like "main/binary-amd64/Packages"
packages_path = "#{component}/binary-#{architecture}/Packages"
# Check if gzip or uncompressed files exist
# (there is no working bzip2 library for Ruby 2.x at the time - 11/2014)
for suffix in ['.gz', '']
packages_path_with_suffix = packages_path+suffix
Rails.logger.debug "Looking for Packages file: #{packages_path_with_suffix}"
url = @dist_url + packages_path_with_suffix
begin
Rails.logger.debug "Try opening URL: #{url}"
file = open(url)
if suffix == '.gz'
file = Zlib::GzipReader.new(file)
end
Rails.logger.debug "File containing packages is: #{file}"
#paragraphs = get_paragraphs(file)
#return paragraphs
#byebug
return get_paragraphs(file)
rescue Errno::ENOENT
Rails.logger.debug "URL could not be opened. Skipping."
end
end
return nil
packages_path = "#{@dist_url}/#{component}/binary-#{architecture}/Packages"
Rails.logger.debug "Loading packages from #{packages_path}"
file = find_and_open_compressed_url(packages_path)
return get_paragraphs(file)
end # def packages
end # class Release
private
# Gather the fields of a Debian control file and return them as a hash
# Gather the fields of a Debian control file section and return them as a hash
def get_fields(data)
fields = {}
@ -81,12 +104,16 @@ module DebImporter
when /^(.+): (.+)/ # "Key: Value"
fields[name.to_sym]=value unless value.empty?
name,value=$1,$2
when /^(.+):\s*/ # "Key:" (start of multi-line entry)
when /^(.+):$/ # "Key:" (start of multi-line entry without value in line)
fields[name.to_sym]=value unless value.empty?
name=$1
value=''
when /^\s+(.+)/ # " Indented multi-line value"
value << $1+"\n"
# Add a newline for multi-line entries ("Key: Value\n Foo\n Bar")
unless value.empty?
value << "\n"
end
value << $1
when /^\s+$/ # Empty line
break
end

View file

@ -36,75 +36,188 @@ BLACKLIST_SECTION_PATTERN = [
/\/?cli-mono$/,
]
# Whether to delete a packag
# Whether to delete a blacklisted package from the database
REMOVE_BLACKLISTED_PACKAGE = true
desc "Import package information from Debian-style Packages files"
namespace :debshots do
desc "Import/update package database from configured DEB repositories"
task :import_debian => :environment do
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
#Rails.logger.level = Logger::DEBUG
task :update_from_deb_repos => :environment do
# Counters for added, updated or removed packages from the database
stats_added = 0
stats_updated = 0
stats_removed = 0
Rails.logger.info "Importing Debian package information"
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
#Rails.logger.level = Logger::DEBUG
repositories.each do |repository|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}"
release = DebImporter::Release.new(repository[:url], repository[:components])
Rails.logger.info "> Supported architectures are: #{release.architectures}"
Rails.logger.info "> Supported components are: #{release.components}"
Rails.logger.info "Importing Debian package information"
# TODO: Remember what we imported to show it on the /about page
repositories.each do |repository|
wanted_architectures = repository[:architectures]
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported architectures are: #{release.architectures}"
if wanted_architectures
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
end
Rails.logger.info "> Supported components are: #{release.components}"
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.architectures.split.each do |architecture|
Rails.logger.info ">> Architecture: #{architecture}"
# Check if this component and architecture is available on this mirror
packages = release.packages(component, architecture)
unless packages
Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
next
end
Rails.logger.info "Packages file for #{component} on #{architecture} found."
Rails.logger.debug "Got: #{packages}"
packages.each do |package|
Rails.logger.info "> Package: #{package[:Package]}"
#Rails.logger.debug "Fetching package informaton from the database"
db_package = Package.find_by name: package[:Package]
if package_blacklisted? package
# Should the package get removed from the database?
if REMOVE_BLACKLISTED_PACKAGE
if db_package
Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
db_package.destroy
end
end
# TODO: Remember what we imported to show it on the /about page
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.architectures.split.each do |architecture|
if wanted_architectures and not wanted_architectures.include?(architecture)
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
next
end # if blacklisted
unless db_package
Rails.logger.debug "No such package in our database."
end
Rails.logger.info ">> Architecture: #{architecture}"
# Check if this component and architecture is available on this mirror
packages = release.packages(component, architecture)
unless packages
Rails.logger.error "No packages for component #{component} on architecture #{architecture} found on this mirror"
next
end
Rails.logger.debug "Found in database: #{db_package.id}"
update_data(package, db_package)
end # package.each
Rails.logger.info "Packages file for #{component} on #{architecture} found."
Rails.logger.debug "Got: #{packages}"
packages.each do |package|
Rails.logger.info "> Package: #{package[:Package]}"
#Rails.logger.debug "Fetching package informaton from the database"
end # architectures.each
end # components.each
if package_name_blacklisted? package[:Package] or package_section_blacklisted? package[:Section]
# Should the package get removed from the database?
if REMOVE_BLACKLISTED_PACKAGE
db_package = Package.find_by name: package[:Package]
if db_package
Rails.logger.info "Removing blacklisted package '#{package[:Package]}' from database"
db_package.destroy
stats_removed +=1
end
end
Rails.logger.info "Done parsing #{repository[:url]} repository."
Rails.logger.info "--------------------------------------------"
end # repositories.each
next
end # if blacklisted
end # task
db_package = Package.find_by name: package[:Package]
unless db_package
Rails.logger.debug "No such package in our database. Creating one."
db_package = Package.create
stats_added += 1
end
# Rails.logger.debug "Found in database: #{db_package}"
update_data(package, db_package)
stats_updated += 1
end # package.each
end # architectures.each
end # components.each
Rails.logger.info "Done parsing #{repository[:url]} repository."
Rails.logger.info "#{stats_added} packages added"
Rails.logger.info "#{stats_updated} packages updated"
Rails.logger.info "#{stats_removed} packages removed"
Rails.logger.info "--------------------------------------------"
end # repositories.each
end # task
desc 'Update long description from i18n file'
task :update_longdescription_from_deb_repos => :environment do
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
# Rails.logger.level = Logger::DEBUG
Rails.logger.info "Importing long descriptoin from Debian repository (i18n)"
repositories.each do |repository|
Rails.logger.info "Fetching Release file for repository: #{repository[:url]} with components: #{repository[:components]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported components are: #{release.components}"
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.i18n(component, 'en').each do |pkg|
Rails.logger.debug "i18n information: #{pkg}"
unless pkg[:'Description-en']
Rails.logger.debug "No Description-en found in section."
next
end
if package_name_blacklisted? pkg[:Package]
next
end
# See if we have that package in the database
if db_pkg = Package.find_by(name: pkg[:Package])
Rails.logger.info "Updating long description for package #{db_pkg.name}"
text = pkg[:'Description-en']
short_description, long_description = text.split("\n", 2)
db_pkg.long_description = long_description
db_pkg.save
else
Rails.logger.debug "Package not in database. Nothing to update."
end
end # pkg.each
end # components.each
Rails.logger.info "Done updating long descriptions from translation file."
Rails.logger.info "--------------------------------------------"
end # repositories.each
end # task
# TODO: Some code duplication. Should be refactord.
desc "List configured DEB repositories"
task :list_deb_repos => :environment do
repositories = Rails.configuration.package_sources
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::INFO
# Rails.logger.level = Logger::DEBUG
Rails.logger.info "Listing configured DEB repositories"
repositories.each do |repository|
wanted_architectures = repository[:architectures]
Rails.logger.info "Fetching Release file for repository: #{repository[:url]}"
release = DebImporter::Release.new(repository[:url])
Rails.logger.info "> Supported architectures are: #{release.architectures}"
if wanted_architectures
Rails.logger.info "> We only want architectures: #{wanted_architectures}"
end
Rails.logger.info "> Supported components are: #{release.components}"
release.components.split.each do |component|
Rails.logger.info "> Component: #{component}"
release.architectures.split.each do |architecture|
if wanted_architectures and not wanted_architectures.include?(architecture)
Rails.logger.debug "Architecture #{architecture} not wanted. Skipping."
next
end
Rails.logger.info ">> Architecture: #{architecture}"
# Check if this component and architecture is available on this mirror
packages = release.packages(component, architecture)
if packages
Rails.logger.info "#{packages.count} packages found."
else
Rails.logger.info "No packages."
end
end # architectures.each
end # components.each
end # repositories.each
end # task
end # namespace
# Update the information about a package in the database
def update_data(package, db_package)
@ -126,33 +239,26 @@ def update_data(package, db_package)
end
db_package.save
# TODO: get the long description from http://de.archive.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
end
# Check if a package is blacklisted and should not be imported.
# Check if a package name is blacklisted and should not be imported.
# This prevents importing boring software that likely does not get a useful screenshot.
# Argument: Hash containing package information from a Packages file
def package_blacklisted?(package)
Rails.logger.debug "Blacklist check of package #{package[:Package]}"
# TODO: add a flag that allows to manually blacklist a package
BLACKLIST_SECTION_PATTERN.each do |pattern|
Rails.logger.debug "Checking package section #{package[:Package]} against pattern #{pattern.inspect}"
if package[:Section] =~ pattern
Rails.logger.info " > Blacklisted by section (matched pattern '#{pattern.inspect}')"
return true
end
def package_name_blacklisted?(name)
if BLACKLIST_NAME_PATTERN.find { |pattern| name=~pattern}
Rails.logger.debug " > Blacklisted by name ('#{name}')"
return true
else
return false
end
end
# Check if a package's section is blacklisted and should not be imported.
# This prevents importing boring software that likely does not get a useful screenshot.
def package_section_blacklisted?(section)
if BLACKLIST_SECTION_PATTERN.find { |pattern| section=~pattern}
Rails.logger.debug " > Blacklisted by section ('#{section}')"
return true
else
return false
end
BLACKLIST_NAME_PATTERN.each do |pattern|
Rails.logger.debug "Checking package name #{package[:Package]} against pattern #{pattern.inspect}"
if package[:Package] =~ pattern
Rails.logger.info " > Blacklisted by name (matched pattern '#{pattern.inspect}')"
return true
end
end
false
end

View file

@ -2,23 +2,23 @@ require 'test_helper'
class PackagesControllerTest < ActionController::TestCase
test "should get index" do
get :index
get :grid
assert_response :success
end
test "should get with_screenshots" do
get :with_screenshots
assert_response :success
end
# test "should get with_screenshots" do
# get :with_screenshots
# assert_response :success
# end
test "should get without_screenshots" do
get :without_screenshots
assert_response :success
end
# test "should get without_screenshots" do
# get :without_screenshots
# assert_response :success
# end
test "should get moderate" do
get :moderate
assert_response :success
end
# test "should get moderate" do
# get :moderate
# assert_response :success
# end
end

View file

@ -2,8 +2,7 @@ require 'test_helper'
class WelcomeControllerTest < ActionController::TestCase
test "should get index" do
get :index
get :home
assert_response :success
end
end

View file

@ -1,23 +1,23 @@
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html
one:
name: MyText
description: MyString
long_description: MyString
section: MyString
maintainer: MyString
maintainer_email: MyString
homepage: MyString
version: MyString
origin: MyString
# one:
# name: MyText
# description: MyString
# long_description: MyString
# section: MyString
# maintainer: MyString
# maintainer_email: MyString
# homepage: MyString
# version: MyString
# origin: MyString
two:
name: MyText
description: MyString
long_description: MyString
section: MyString
maintainer: MyString
maintainer_email: MyString
homepage: MyString
version: MyString
origin: MyString
# two:
# name: MyText
# description: MyString
# long_description: MyString
# section: MyString
# maintainer: MyString
# maintainer_email: MyString
# homepage: MyString
# version: MyString
# origin: MyString

View file

@ -1,21 +1,21 @@
# Read about fixtures at http://api.rubyonrails.org/classes/ActiveRecord/Fixtures.html
one:
version: MyString
uploaddatetime: 2013-07-27 13:29:55
uploaderhash: MyString
uploaderip: MyString
approved:
markedfordelete:
delete_reason: MyString
description: MyString
# one:
# version: MyString
# uploaddatetime: 2013-07-27 13:29:55
# uploaderhash: MyString
# uploaderip: MyString
# approved:
# markedfordelete:
# delete_reason: MyString
# description: MyString
two:
version: MyString
uploaddatetime: 2013-07-27 13:29:55
uploaderhash: MyString
uploaderip: MyString
approved:
markedfordelete:
delete_reason: MyString
description: MyString
# two:
# version: MyString
# uploaddatetime: 2013-07-27 13:29:55
# uploaderhash: MyString
# uploaderip: MyString
# approved:
# markedfordelete:
# delete_reason: MyString
# description: MyString

View file

@ -0,0 +1,40 @@
$LOAD_PATH << 'lib'
require 'open-uri' # allows to load URLs using open()
require 'deb_importer'
require 'test_helper'
include DebImporter
class PackagesHelperTest < ActionView::TestCase
test "should be able to parse local Debian repository test files" do
Rails.logger = Logger.new(STDOUT)
Rails.logger.level = Logger::ERROR
# Rails.logger.level = Logger::INFO
# Rails.logger.level = Logger::DEBUG
# Load information about test repository from environments/test.rb
repositories = Rails.configuration.package_sources
Rails.logger.info "Test parsing local DEB-style repository"
repository = repositories.first
release = DebImporter::Release.new(repository[:url])
assert_equal release.components, 'main restricted universe multiverse'
# Packages enumerator must be restarted to start from position 1
packages = release.packages('main', 'amd64')
assert_instance_of Enumerator, packages
first_package = packages.first
assert_equal 'account-plugin-aim', first_package[:Package]
# Packages enumerator must be restarted to start from position 1
packages = release.packages('main', 'amd64')
assert_equal 8320, packages.count
# Get information from translation file
i18n = release.i18n('main', 'en')
assert_match /^Real-time strategy game of ancient warfare\n0 A\.D\./, i18n.first[:'Description-en']
end # test
end # class

View file

@ -1,15 +1,17 @@
ENV["RAILS_ENV"] ||= "test"
require File.expand_path('../../config/environment', __FILE__)
require 'rails/test_help'
ENV['RAILS_ENV'] ||= 'test'
require File.expand_path("../../config/environment", __FILE__)
require "rails/test_help"
# require "minitest/rails"
# require "minitest/autorun"
# To add Capybara feature tests add `gem "minitest-rails-capybara"`
# to the test group in the Gemfile and uncomment the following:
# require "minitest/rails/capybara"
# Uncomment for awesome colorful output
# require "minitest/pride"
class ActiveSupport::TestCase
ActiveRecord::Migration.check_pending!
# Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
#
# Note: You'll currently still have to declare fixtures explicitly in integration tests
# -- they do not yet inherit this setting
fixtures :all
# Add more helper methods to be used by all tests here...
# Setup all fixtures in test/fixtures/*.(yml|csv) for all tests in alphabetical order.
fixtures :all
end