449 lines
14 KiB
Text
449 lines
14 KiB
Text
# Installing debshots the easy way using Ansible
|
|
|
|
Prerequisites: A Debian Jessie server.
|
|
|
|
Install PIP to get a recent version of Ansible:
|
|
|
|
apt install python-pip
|
|
|
|
Tun the Ansible playbook that comes with the debshots repository:
|
|
|
|
cd ansible
|
|
ansible-playbook debshost.yml
|
|
|
|
_(If you get "command not found" then run "hash -r" or "rehash" in your
|
|
shell and try again.)_
|
|
|
|
|
|
# Installing debshots the manual way
|
|
|
|
## Prepare PostgreSQL database server and user (ident-based authentication)
|
|
|
|
apt install postgresql
|
|
su - postgres
|
|
createuser -P -d debshots
|
|
createdb -O debshots -E UTF8 -T template0 debshots
|
|
|
|
## Create an application user
|
|
|
|
adduser debshots
|
|
|
|
Allow the user to parse access log files from Nginx:
|
|
|
|
adduser debshots adm
|
|
su - debshots
|
|
|
|
## Install the required Ruby version using rbenv
|
|
|
|
Documentation: https://github.com/rbenv/rbenv
|
|
|
|
git clone https://github.com/rbenv/rbenv.git ~/.rbenv
|
|
echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile
|
|
|
|
# Load rbenv automatically by appending
|
|
# the following to ~/.profile:
|
|
|
|
eval "$(rbenv init -)"
|
|
|
|
Install the "rbenv install" add-on:
|
|
|
|
Documentation: https://github.com/rbenv/ruby-build#readme
|
|
|
|
git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build
|
|
|
|
Install the "rbenv vars" add-on:
|
|
|
|
cd ~/.rbenv/plugins
|
|
git clone https://github.com/sstephenson/rbenv-vars.git
|
|
|
|
## Get the application
|
|
|
|
Clone the debshots Git repository.
|
|
|
|
## Install the Ruby dependencies
|
|
|
|
rbenv install
|
|
echo "gem: --no-rdoc --no-ri" > ~/.gemrc
|
|
gem install bundler
|
|
bundle install --deployment
|
|
|
|
## Optional: copy screenshots and database from the live website
|
|
|
|
scp screenshots.debian.net:debshots-screenshots.tar .
|
|
tar -C public/ -xvf debshots-screenshots.tar
|
|
scp screenshots.debian.net:debshots.sql .
|
|
psql debshots < debshots.sql
|
|
|
|
## Switch to production environment
|
|
|
|
export RAILS_ENV=production
|
|
|
|
## Migrate database
|
|
|
|
bundle exec rake db:migrate RAILS_ENV=production
|
|
|
|
In config/database.yml: production: user/password/host must be commented out
|
|
|
|
## Pre-render the static assets
|
|
|
|
bundle exec rake assets:precompile RAILS_ENV=production
|
|
|
|
## Test the application:
|
|
|
|
bundle exec rails s -b 0.0.0.0 -e production
|
|
|
|
Point your browser to http://...:3000/ and check the web site.
|
|
|
|
## Create a user for moderation:
|
|
|
|
Start a Rails console:
|
|
|
|
bundle exec rails c -e production
|
|
|
|
Create a new user record:
|
|
|
|
user = User.create(realname: 'Christoph Haas', password: 'foobartest', email: 'email@christoph-haas.de')
|
|
user.save!
|
|
|
|
## Tidy up the screenshot data
|
|
|
|
bundle exec rake debshots:remove_broken_screenshots
|
|
bundle exec rake debshots:remove_duplicate_images
|
|
|
|
## Import new package information
|
|
|
|
bundle exec rake debshots:list_deb_repos
|
|
bundle exec rake debshots:update_from_deb_repos
|
|
bundle exec debshots:update_longdescription_from_deb_repos
|
|
|
|
|
|
|
|
# Prepare production environment
|
|
|
|
## Install passenger to run the application behind nginx
|
|
|
|
Use the Ansible playbook to install Phusion Passenger - the component
|
|
that makes the actual Ruby on Rails web application accessible from
|
|
the Nginx web server.
|
|
|
|
## Example nginx vhost config
|
|
|
|
more_clear_headers 'X-Runtime';
|
|
more_clear_headers 'X-Powered-By';
|
|
more_clear_headers 'Server';
|
|
server_tokens off;
|
|
|
|
proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off;
|
|
|
|
log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $remote_addr $request_uri "$http_referer" "$http_user_agent" $scheme';
|
|
|
|
# Performance
|
|
passenger_max_pool_size 8;
|
|
passenger_max_request_queue_size 200;
|
|
|
|
limit_req_zone $binary_remote_addr zone=one:20m rate=5r/s;
|
|
|
|
# Frontend caching and static asset delivery
|
|
server {
|
|
listen 85.25.83.22:80 default_server;
|
|
listen 85.25.83.22:443 ssl;
|
|
ssl_certificate /etc/letsencrypt/live/screenshots.debian.net/fullchain.pem;
|
|
ssl_certificate_key /etc/letsencrypt/live/screenshots.debian.net/privkey.pem;
|
|
|
|
root /home/debshots/debshots/public;
|
|
server_name _;
|
|
access_log /var/log/nginx/cache-access.log proxy;
|
|
|
|
add_header X-Cache-Status $upstream_cache_status;
|
|
|
|
# Tell the backend if the protocol used was HTTPS. Otherwise you get an infinite
|
|
# redirection loop because the backend assumes that HTTP was spoken behind the
|
|
# proxy.
|
|
proxy_set_header X-Forwarded-Proto $scheme;
|
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
|
|
# Pass on the actual HTTP_HOST ("Host:" header) so that Rails can build proper absolute URLs
|
|
proxy_set_header Host $host;
|
|
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
|
|
gzip_proxied any;
|
|
|
|
location / {
|
|
try_files /maintenance.html @backend;
|
|
add_header X-Coffee front-main;
|
|
}
|
|
|
|
location /json/ {
|
|
try_files /maintenance.html @backend;
|
|
proxy_cache my_cache;
|
|
proxy_pass http://127.0.0.1:8888/;
|
|
proxy_cache_lock on;
|
|
proxy_cache_use_stale updating;
|
|
add_header X-Coffee front-json;
|
|
}
|
|
|
|
location /favicon.ico {
|
|
alias /home/debshots/debshots/public/favicon.ico;
|
|
}
|
|
|
|
location @backend {
|
|
proxy_cache my_cache;
|
|
proxy_pass http://127.0.0.1:8888;
|
|
|
|
# Avoid cache stampede - rather update once and deliver stale content
|
|
proxy_cache_lock on;
|
|
proxy_cache_use_stale updating;
|
|
add_header X-Coffee front-rails;
|
|
}
|
|
|
|
location /assets/ {
|
|
alias /home/debshots/debshots/public/assets/;
|
|
expires 1h;
|
|
add_header X-Coffee front-assets;
|
|
}
|
|
location /logo/ {
|
|
alias /home/debshots/debshots/public/logo/;
|
|
expires 1h;
|
|
add_header X-Coffee front-logo;
|
|
}
|
|
location /screenshots/ {
|
|
alias /home/debshots/debshots/public/screenshots/;
|
|
expires max;
|
|
add_header X-Coffee front-screenshots;
|
|
}
|
|
location /images/ {
|
|
alias /home/debshots/debshots/public/images/;
|
|
expires 1h;
|
|
add_header X-Coffee front-images;
|
|
}
|
|
}
|
|
|
|
# Backend rails application
|
|
server {
|
|
listen 127.0.0.1:8888;
|
|
|
|
root /home/debshots/debshots/public;
|
|
|
|
server_name _;
|
|
access_log /var/log/nginx/rails-access.log;
|
|
|
|
passenger_enabled on;
|
|
passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby;
|
|
|
|
|
|
# Send thumbnails using X-Sendfile / X-Accel-Redirect
|
|
# The correct thumbnail is computed by the Rails application so it cannot be served directly.
|
|
location /thumbnail/ {
|
|
expires 1h;
|
|
proxy_cache_valid 404 15m;
|
|
add_header X-Coffee back-thumbnail;
|
|
limit_req zone=one burst=30;
|
|
}
|
|
|
|
# Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...)
|
|
location /public/ {
|
|
expires 1h;
|
|
add_header X-Coffee back-public;
|
|
|
|
passenger_set_header X-Sendfile-Type X-Accel-Redirect;
|
|
passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/;
|
|
passenger_pass_header X-Accel-Redirect;
|
|
}
|
|
|
|
location /__send_file_accel/ {
|
|
internal;
|
|
alias /home/debshots/debshots/public/;
|
|
add_header X-Coffee back-accel;
|
|
expires 1d;
|
|
more_clear_headers 'Set-Cookie';
|
|
}
|
|
|
|
location /secretstatus {
|
|
stub_status;
|
|
access_log off;
|
|
allow all;
|
|
add_header X-Coffee back-status;
|
|
}
|
|
}
|
|
|
|
|
|
# Supported URL paths (aka routes)
|
|
|
|
_For up-to-date information please check config/routes._
|
|
|
|
/screenshots/:package_inital/:package/:(id)_:size.png
|
|
* package = name of the package
|
|
* id = numerical ID of the screenshot
|
|
* size = "large" or "small"
|
|
* Returns a PNG image of a screenshot
|
|
* Should be served as a static asset
|
|
|
|
/image/:(id)_:size.png
|
|
* id = numerical ID of the screenshot
|
|
* size = "large" or "small"
|
|
* Returns a PNG image of a screenshot
|
|
* Usually used for unapproved images that should just be shown to the uploader
|
|
|
|
/about
|
|
* Return an informational HTML page
|
|
|
|
/packages
|
|
* Return a list of packages with pagination
|
|
|
|
/packages/with_screenshots
|
|
* Return a list of packages with pagination
|
|
* Select only those packages that have screenshots.
|
|
|
|
/packages/without_screenshots
|
|
* Return a list of packages with pagination
|
|
* Select only those packages that do not have screenshots.
|
|
|
|
/packages/games_without_screenshots (deprecated)
|
|
* Return a list of packages with pagination
|
|
* Select only those packages that belong to Debian's "games" section
|
|
|
|
/json/packages
|
|
* Return a JSON data structure of all packages
|
|
|
|
/json/screenshots
|
|
* Return a JSON data structure containing packages and screenshots
|
|
|
|
/json/packages-without-screenshots
|
|
* Return a JSON data structure of those packages that do not have any screenshots
|
|
|
|
/packages/moderate
|
|
* Return a HTML page showing packages that need moderation
|
|
* Must only be available to logged in moderators
|
|
|
|
/upload
|
|
* Return a HTML page containing an upload form for new screenshots
|
|
|
|
/guidelines (deprecated)
|
|
* Return a HTML page showing general guidelines for creating screenshots
|
|
* Should just redirect to the /upload page
|
|
|
|
/upload/:package
|
|
* ??
|
|
|
|
/uploadfile
|
|
* ??
|
|
|
|
/login
|
|
* Show a login form
|
|
|
|
/logout
|
|
* Logout the user by invalidating the cookie session.
|
|
|
|
/package/:package
|
|
* Show details of a certain package
|
|
|
|
/json/package/:package
|
|
* Return a JSON data structure containing details of a certain package
|
|
|
|
/thumbnail/:package
|
|
/thumbnail-404/:package (deprecated)
|
|
* Return a small (160x120 pixel) PNG screenshot file of a package
|
|
* If the package does not have any screenshots then return a 404 code
|
|
|
|
/thumbnail-with-version/:package/:version
|
|
* Return a small (160x120 pixel) PNG screenshot file of a package
|
|
* If the package does not have any screenshots then return a 404 code
|
|
* Prefer screenshots of a certain version
|
|
|
|
/screenshot/:package
|
|
/screenshot-404/:package (deprecated)
|
|
* Return a large (320x240 pixel) PNG screenshot file of a package
|
|
* If the package does not have any screenshots then return a 404 code
|
|
|
|
/screenshot-with-version/:package/:version
|
|
* Return a large (320x240 pixel) PNG screenshot file of a package
|
|
* If the package does not have any screenshots then return a 404 code
|
|
* Prefer screenshots of a certain version
|
|
|
|
/delete_screenshot/:screenshot
|
|
* Users: request deletion of a screenshot
|
|
* Moderators: delete a screenshot
|
|
|
|
/approve_screenshot/:screenshot
|
|
* Users: 403
|
|
* Moderators: approve a screenshot
|
|
|
|
/keep_screenshot/:screenshot
|
|
* Users: 403
|
|
* Moderators: re-approve a screenshot that was requested for deletion
|
|
|
|
## Cron jobs
|
|
|
|
Parse Nginx access logs to count the number of visits of each package:
|
|
|
|
RAILS_ENV=production bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1]
|
|
|
|
As a cron job:
|
|
|
|
41 9 * * * cd $HOME/debshots ; RAILS_ENV=production /home/debshots/.rbenv/shims/bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1]
|
|
|
|
|
|
# Authentication
|
|
|
|
This web application tries to avoid to create yet another directory
|
|
of user accounts. So it leverages other common web sites like StackExchange,
|
|
Debian SSO (using browser/client certificates) or GitHub as single-sign-on
|
|
services. It is still possible to create local user accounts for administrators
|
|
but registration is disabled.
|
|
|
|
Anonymous uploads are possible but they have to be moderated by an administrator.
|
|
|
|
# Authorization
|
|
|
|
A user can have one of these access levels.
|
|
|
|
## Anonymous
|
|
|
|
Without authentication all user actions are moderated. Such users can upload
|
|
screenshots or report existing/published screenshots as inappropriate. However
|
|
a user with level 'admin' will have to approve that action.
|
|
|
|
## Authenticated by single-sign-login (SSO)
|
|
|
|
Users can use different authentication services to login - like Github or
|
|
StackExchange. Such users internally get an account with a random password
|
|
created but they will keep logging in via SSO.
|
|
|
|
The first upload will have to be moderated. Subsequent uploads will be
|
|
approved automatically.
|
|
|
|
## Authenticated by Debian signle-sign-login (SSO)
|
|
|
|
If a user authenticates using a Debian SSO client certificate they will be
|
|
able to upload screenshots without moderation.
|
|
(Anohter idea was to restrict them to uploading only screenshots for the
|
|
packages they maintain. But that may be reliable and is not implemented at
|
|
this time.)
|
|
|
|
## Administrator
|
|
|
|
Such a user has an internal user account. The database record has the
|
|
'admin' field set to 1. Admins can freely upload, moderate and delete
|
|
screenshots. They can also read the log files.
|
|
|
|
|
|
# Nginx during development
|
|
|
|
During development it is advised to run Nginx to proxy ports 3000 (HTTP)
|
|
and 3001 (HTTPS) to the rails application.
|
|
|
|
cd nginx
|
|
nginx -c `pwd`/nginx.conf
|
|
|
|
The SSL certitificate is just a self-signed certificate with the
|
|
common name "localhost". It was generated using:
|
|
|
|
openssl req -x509 -newkey 4096 -nodes -keyout localhost.key -out localhost.crt -days 3650
|
|
|
|
Run "rails s" in another window. That will use the built-in Ruby 'Puma'
|
|
application server that can also be used in production. It requires less
|
|
configuration than Phusion Passenger but may be slower in regard to
|
|
performance and multi-threading / multi-core support.
|
|
|
|
If you get "400 Bad Request / The SSL certificate error" in the browser then
|
|
please check the error log (/tmp/error*). Once cause is that the CRL of
|
|
the Debian SSO has expired and needs to be updated.
|