diff --git a/doc/README.DeveloperNotes b/doc/README.DeveloperNotes new file mode 100644 index 0000000..407991e --- /dev/null +++ b/doc/README.DeveloperNotes @@ -0,0 +1,93 @@ +# Full text search + +The pg_search Ruby gem is used for full-text searched in Rails. +See: https://github.com/Casecommons/pg_search + +Reformatted SQL query for fulltext search: + +SELECT COUNT(*) FROM "packages" WHERE ( + ( + ( + to_tsvector('simple', coalesce("packages"."name"::text, '')) + || + to_tsvector('simple', coalesce("packages"."description"::text, '')) + || + to_tsvector('simple', coalesce("packages"."long_description"::text, '')) + ) + @@ + ( + to_tsquery('simple', '''' || 'browser' || ' ''') + ) + ) +) + +# Index: + +CREATE INDEX packages_gin ON packages USING GIN( (to_tsvector('simple', +coalesce("packages"."name"::text, '')) || to_tsvector('simple', +coalesce("packages"."description"::text, '')) || to_tsvector('simple', +coalesce("packages"."long_description"::text, ''))) ); + +--------- + +Used: https://github.com/semaperepelitsa/jquery.fileupload-rails +Which is a gem for: https://github.com/blueimp/jQuery-File-Upload + +Paperclip for attachment/screenshot file handlind within ActiveRecord +https://github.com/thoughtbot/paperclip + +-- + +Components: + +- Rails 5 (widespread, up-to-date, solves many common problems) +- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search) +- Git (most people nowadays use Git) +- jQuery (well known and easy to use) +- Zurb Foundation (integrates nicer than Twitter Bootstrap) +- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails) + +-- + +Importer + +How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat + +-- + +Logging from within the application into the database's "log" table: + +Log.log "foo" +or with section (default='frontend'): +Log.log "foo", "importer" + +-- + +To create admin users +--------------------- + +rails c +> User.new(email: 'email@christoph-haas.de', password: 'secret').save + +--- + +Multiple distributions (e.g. Ubuntu, Debian) +- new table: distributions + - id + - name (e.g. "Debian") + - description (e.g. "The Debian GNU/Linux distribution") + - url (http://www.debian.org/) + - type (apt, later: rpm) + - logo + +- screenshots: new field for source_id + +Admin interface to manage sources + +--- + +AppStream data +- screenshots: new field for type ('user-generated', 'upstream', 'logo') + user-generated: uploaded by a random user + upstream: official screenshot from the developer + logo: not a screenshot - just a logo diff --git a/doc/README.Development b/doc/README.Development new file mode 100644 index 0000000..2caf479 --- /dev/null +++ b/doc/README.Development @@ -0,0 +1,5 @@ +To run the Rails server in development mode on port 3001 with HTTPS: + + rails s puma -b 'ssl://0.0.0.0:3001?key=nginx/localhost.key&cert=nginx/localhost.crt&verify_mode=none' + +This is required to be able to use single-sign login with the different services. diff --git a/doc/README.Installation b/doc/README.Installation new file mode 100644 index 0000000..12a1fc0 --- /dev/null +++ b/doc/README.Installation @@ -0,0 +1,449 @@ +# Installing debshots the easy way using Ansible + +Prerequisites: A Debian Jessie server. + +Install PIP to get a recent version of Ansible: + + apt install python-pip + +Tun the Ansible playbook that comes with the debshots repository: + + cd ansible + ansible-playbook debshost.yml + +_(If you get "command not found" then run "hash -r" or "rehash" in your +shell and try again.)_ + + +# Installing debshots the manual way + +## Prepare PostgreSQL database server and user (ident-based authentication) + + apt install postgresql + su - postgres + createuser -P -d debshots + createdb -O debshots -E UTF8 -T template0 debshots + +## Create an application user + + adduser debshots + +Allow the user to parse access log files from Nginx: + + adduser debshots adm + su - debshots + +## Install the required Ruby version using rbenv + +Documentation: https://github.com/rbenv/rbenv + + git clone https://github.com/rbenv/rbenv.git ~/.rbenv + echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile + +# Load rbenv automatically by appending +# the following to ~/.profile: + + eval "$(rbenv init -)" + +Install the "rbenv install" add-on: + +Documentation: https://github.com/rbenv/ruby-build#readme + + git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build + +Install the "rbenv vars" add-on: + + cd ~/.rbenv/plugins + git clone https://github.com/sstephenson/rbenv-vars.git + +## Get the application + +Clone the debshots Git repository. + +## Install the Ruby dependencies + + rbenv install + echo "gem: --no-rdoc --no-ri" > ~/.gemrc + gem install bundler + bundle install --deployment + +## Optional: copy screenshots and database from the live website + + scp screenshots.debian.net:debshots-screenshots.tar . + tar -C public/ -xvf debshots-screenshots.tar + scp screenshots.debian.net:debshots.sql . + psql debshots < debshots.sql + +## Switch to production environment + + export RAILS_ENV=production + +## Migrate database + + bundle exec rake db:migrate RAILS_ENV=production + +In config/database.yml: production: user/password/host must be commented out + +## Pre-render the static assets + + bundle exec rake assets:precompile RAILS_ENV=production + +## Test the application: + + bundle exec rails s -b 0.0.0.0 -e production + +Point your browser to http://...:3000/ and check the web site. + +## Create a user for moderation: + +Start a Rails console: + + bundle exec rails c -e production + +Create a new user record: + + user = User.create(realname: 'Christoph Haas', password: 'foobartest', email: 'email@christoph-haas.de') + user.save! + +## Tidy up the screenshot data + + bundle exec rake debshots:remove_broken_screenshots + bundle exec rake debshots:remove_duplicate_images + +## Import new package information + + bundle exec rake debshots:list_deb_repos + bundle exec rake debshots:update_from_deb_repos + bundle exec debshots:update_longdescription_from_deb_repos + + + +# Prepare production environment + +## Install passenger to run the application behind nginx + +Use the Ansible playbook to install Phusion Passenger - the component +that makes the actual Ruby on Rails web application accessible from +the Nginx web server. + +## Example nginx vhost config + + more_clear_headers 'X-Runtime'; + more_clear_headers 'X-Powered-By'; + more_clear_headers 'Server'; + server_tokens off; + + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off; + + log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $remote_addr $request_uri "$http_referer" "$http_user_agent" $scheme'; + + # Performance + passenger_max_pool_size 8; + passenger_max_request_queue_size 200; + + limit_req_zone $binary_remote_addr zone=one:20m rate=5r/s; + + # Frontend caching and static asset delivery + server { + listen 85.25.83.22:80 default_server; + listen 85.25.83.22:443 ssl; + ssl_certificate /etc/letsencrypt/live/screenshots.debian.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/screenshots.debian.net/privkey.pem; + + root /home/debshots/debshots/public; + server_name _; + access_log /var/log/nginx/cache-access.log proxy; + + add_header X-Cache-Status $upstream_cache_status; + + # Tell the backend if the protocol used was HTTPS. Otherwise you get an infinite + # redirection loop because the backend assumes that HTTP was spoken behind the + # proxy. + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Pass on the actual HTTP_HOST ("Host:" header) so that Rails can build proper absolute URLs + proxy_set_header Host $host; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + gzip_proxied any; + + location / { + try_files /maintenance.html @backend; + add_header X-Coffee front-main; + } + + location /json/ { + try_files /maintenance.html @backend; + proxy_cache my_cache; + proxy_pass http://127.0.0.1:8888/; + proxy_cache_lock on; + proxy_cache_use_stale updating; + add_header X-Coffee front-json; + } + + location /favicon.ico { + alias /home/debshots/debshots/public/favicon.ico; + } + + location @backend { + proxy_cache my_cache; + proxy_pass http://127.0.0.1:8888; + + # Avoid cache stampede - rather update once and deliver stale content + proxy_cache_lock on; + proxy_cache_use_stale updating; + add_header X-Coffee front-rails; + } + + location /assets/ { + alias /home/debshots/debshots/public/assets/; + expires 1h; + add_header X-Coffee front-assets; + } + location /logo/ { + alias /home/debshots/debshots/public/logo/; + expires 1h; + add_header X-Coffee front-logo; + } + location /screenshots/ { + alias /home/debshots/debshots/public/screenshots/; + expires max; + add_header X-Coffee front-screenshots; + } + location /images/ { + alias /home/debshots/debshots/public/images/; + expires 1h; + add_header X-Coffee front-images; + } + } + + # Backend rails application + server { + listen 127.0.0.1:8888; + + root /home/debshots/debshots/public; + + server_name _; + access_log /var/log/nginx/rails-access.log; + + passenger_enabled on; + passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; + + + # Send thumbnails using X-Sendfile / X-Accel-Redirect + # The correct thumbnail is computed by the Rails application so it cannot be served directly. + location /thumbnail/ { + expires 1h; + proxy_cache_valid 404 15m; + add_header X-Coffee back-thumbnail; + limit_req zone=one burst=30; + } + + # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) + location /public/ { + expires 1h; + add_header X-Coffee back-public; + + passenger_set_header X-Sendfile-Type X-Accel-Redirect; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + location /__send_file_accel/ { + internal; + alias /home/debshots/debshots/public/; + add_header X-Coffee back-accel; + expires 1d; + more_clear_headers 'Set-Cookie'; + } + + location /secretstatus { + stub_status; + access_log off; + allow all; + add_header X-Coffee back-status; + } + } + + +# Supported URL paths (aka routes) + +_For up-to-date information please check config/routes._ + +/screenshots/:package_inital/:package/:(id)_:size.png +* package = name of the package +* id = numerical ID of the screenshot +* size = "large" or "small" +* Returns a PNG image of a screenshot +* Should be served as a static asset + +/image/:(id)_:size.png +* id = numerical ID of the screenshot +* size = "large" or "small" +* Returns a PNG image of a screenshot +* Usually used for unapproved images that should just be shown to the uploader + +/about +* Return an informational HTML page + +/packages +* Return a list of packages with pagination + +/packages/with_screenshots +* Return a list of packages with pagination +* Select only those packages that have screenshots. + +/packages/without_screenshots +* Return a list of packages with pagination +* Select only those packages that do not have screenshots. + +/packages/games_without_screenshots (deprecated) +* Return a list of packages with pagination +* Select only those packages that belong to Debian's "games" section + +/json/packages +* Return a JSON data structure of all packages + +/json/screenshots +* Return a JSON data structure containing packages and screenshots + +/json/packages-without-screenshots +* Return a JSON data structure of those packages that do not have any screenshots + +/packages/moderate +* Return a HTML page showing packages that need moderation +* Must only be available to logged in moderators + +/upload +* Return a HTML page containing an upload form for new screenshots + +/guidelines (deprecated) +* Return a HTML page showing general guidelines for creating screenshots +* Should just redirect to the /upload page + +/upload/:package +* ?? + +/uploadfile +* ?? + +/login +* Show a login form + +/logout +* Logout the user by invalidating the cookie session. + +/package/:package +* Show details of a certain package + +/json/package/:package +* Return a JSON data structure containing details of a certain package + +/thumbnail/:package +/thumbnail-404/:package (deprecated) +* Return a small (160x120 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code + +/thumbnail-with-version/:package/:version +* Return a small (160x120 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code +* Prefer screenshots of a certain version + +/screenshot/:package +/screenshot-404/:package (deprecated) +* Return a large (320x240 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code + +/screenshot-with-version/:package/:version +* Return a large (320x240 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code +* Prefer screenshots of a certain version + +/delete_screenshot/:screenshot +* Users: request deletion of a screenshot +* Moderators: delete a screenshot + +/approve_screenshot/:screenshot +* Users: 403 +* Moderators: approve a screenshot + +/keep_screenshot/:screenshot +* Users: 403 +* Moderators: re-approve a screenshot that was requested for deletion + +## Cron jobs + +Parse Nginx access logs to count the number of visits of each package: + + RAILS_ENV=production bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] + +As a cron job: + + 41 9 * * * cd $HOME/debshots ; RAILS_ENV=production /home/debshots/.rbenv/shims/bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] + + +# Authentication + +This web application tries to avoid to create yet another directory +of user accounts. So it leverages other common web sites like StackExchange, +Debian SSO (using browser/client certificates) or GitHub as single-sign-on +services. It is still possible to create local user accounts for administrators +but registration is disabled. + +Anonymous uploads are possible but they have to be moderated by an administrator. + +# Authorization + +A user can have one of these access levels. + +## Anonymous + +Without authentication all user actions are moderated. Such users can upload +screenshots or report existing/published screenshots as inappropriate. However +a user with level 'admin' will have to approve that action. + +## Authenticated by single-sign-login (SSO) + +Users can use different authentication services to login - like Github or +StackExchange. Such users internally get an account with a random password +created but they will keep logging in via SSO. + +The first upload will have to be moderated. Subsequent uploads will be +approved automatically. + +## Authenticated by Debian signle-sign-login (SSO) + +If a user authenticates using a Debian SSO client certificate they will be +able to upload screenshots without moderation. +(Anohter idea was to restrict them to uploading only screenshots for the +packages they maintain. But that may be reliable and is not implemented at +this time.) + +## Administrator + +Such a user has an internal user account. The database record has the +'admin' field set to 1. Admins can freely upload, moderate and delete +screenshots. They can also read the log files. + + +# Nginx during development + +During development it is advised to run Nginx to proxy ports 3000 (HTTP) +and 3001 (HTTPS) to the rails application. + + cd nginx + nginx -c `pwd`/nginx.conf + +The SSL certitificate is just a self-signed certificate with the +common name "localhost". It was generated using: + + openssl req -x509 -newkey 4096 -nodes -keyout localhost.key -out localhost.crt -days 3650 + +Run "rails s" in another window. That will use the built-in Ruby 'Puma' +application server that can also be used in production. It requires less +configuration than Phusion Passenger but may be slower in regard to +performance and multi-threading / multi-core support. + +If you get "400 Bad Request / The SSL certificate error" in the browser then +please check the error log (/tmp/error*). Once cause is that the CRL of +the Debian SSO has expired and needs to be updated. diff --git a/doc/README.authentication.md b/doc/README.authentication.md new file mode 100644 index 0000000..dd912f2 --- /dev/null +++ b/doc/README.authentication.md @@ -0,0 +1,92 @@ +# Authentication + +The application uses _omniauth_ to allow local authentication for moderators +and adds OAuth authentication to connect to Google+, Github and other +authentication providers. + +Documentation on omniauth: + +https://github.com/omniauth/omniauth/wiki/List-of-Strategies +https://github.com/plataformatec/devise/wiki/OmniAuth:-Overview + +List of strategies: + +https://github.com/omniauth/omniauth/wiki/List-of-Strategies + +## Google + +The client ID for the debshots and the private key must be set as environment +variables before running the application: + +* GOOGLE_CLIENT_ID=… +* GOOGLE_CLIENT_SECRET=… + +To manage OAuth applications at Google: https://console.developers.google.com/apis/credentials + +Authorized callback URLs must be set to: + +- https://screenshots.debian.net/users/auth/google_oauth2/callback +- https://localhost:3001/users/auth/google_oauth2/callback + +## Amazon + +The client ID for the debshots and the secret must be set as environment +variables before running the application: + +* AMAZON_CLIENT_ID=… +* AMAZON_CLIENT_SECRET=… + + +Client ID: + + amzn1.application-oa2-client.78d832919fc24456b0636762cf18efd9 + +Client Secret: + + 8c495d0940ca4100313c03e93b8b4240eef256d2fdbe672718506f29f5a20f61 + +Available login buttons: https://login.amazon.com/button-guide + +## Launchpad + +TODO + +## Github + +TODO + +## Caveats + +The OAuth callback handling uses Ruby's "faraday" library. Unfortunately +Faraday may easily miss the directory where the common CAs are stored. +So the path to the certificates on the system needs to be provided: + + config.omniauth :google_oauth2, '398593918966-0fpmit0pb6ptio1ndsie5dfcnqhei63l.apps.googleusercontent.com', 'uRQq_dLs1kx7l5sYIyEyPVRc', :client_options => {:ssl => {:ca_path => '/etc/ssl/certs'}} + +For another explanation of the issue see: https://gist.github.com/kt103099/3183125 + +## Debian + +The Debian SSO service uses certificates to provide access for Debian developers. +If the frontend web server (e.g. nginx) is told to allow optional authentication +using SSL client certificates then a visitor gets requested to send a certificates +if one is installed in the browser. Otherwise users just do not notice anything. + +Authentication can be tested by: + + cd nginx + nginx -c `pwd`/nginx.conf + +(Ignore the warning about the error log file.) + +Then point your browser to: https://localhost:3001/ + +The relevant part of the Nginx configuration: + + ssl_client_certificate debsso.crt; + ssl_crl debsso.crl; + ssl_verify_client optional; + +However the CRL (certificate revocation list) needs to be updated regularly. +The appropriate update script can be found at: +https://anonscm.debian.org/cgit/debian-sso/debian-sso.git/plain/update-debsso-ca