diff --git a/doc/README.rdoc b/doc/README.rdoc index b011d34..4017cf8 100644 --- a/doc/README.rdoc +++ b/doc/README.rdoc @@ -1,244 +1,271 @@ -== Installing debshots +# Installing debshots the easy way using Ansible -=== Prepare PostgreSQL database server and user (ident-based authentication) +Prerequisites: A Debian Jessie server. -apt install postgresql +Install PIP to get a recent version of Ansible: -su - postgres + apt install python-pip -createuser -d debshots +Tun the Ansible playbook that comes with the debshots repository: -createdb -O debshots -E UTF8 -T template0 debshots + cd ansible + ansible-playbook debshost.yml -=== Create an application user +_(If you get "command not found" then run "hash -r" or "rehash" in your +shell and try again.)_ -adduser debshots + + + +# Installing debshots the manual way + +## Prepare PostgreSQL database server and user (ident-based authentication) + + apt install postgresql + su - postgres + createuser -d debshots + createdb -O debshots -E UTF8 -T template0 debshots + +## Create an application user + + adduser debshots Allow the user to parse access log files from Nginx: -adduser debshots adm + adduser debshots adm + su - debshots -su - debshots +## Install the required Ruby version using rbenv -=== Install the required Ruby version using rbenv - -https://github.com/rbenv/rbenv - -git clone https://github.com/rbenv/rbenv.git ~/.rbenv - -echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile +Documentation: https://github.com/rbenv/rbenv + git clone https://github.com/rbenv/rbenv.git ~/.rbenv + echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile # Load rbenv automatically by appending -# the following to ~/.bashrc: +# the following to ~/.profile: -eval "$(rbenv init -)" + eval "$(rbenv init -)" +Install the "rbenv install" add-on: -https://github.com/rbenv/ruby-build#readme +Documentation: https://github.com/rbenv/ruby-build#readme -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build + git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -=== Get the application +## Get the application Clone the debshots Git repository. -=== Install the Ruby dependencies +## Install the Ruby dependencies -rbenv install + rbenv install + echo "gem: --no-rdoc --no-ri" > ~/.gemrc + gem install bundler + bundle install --deployment -echo "gem: --no-rdoc --no-ri" > ~/.gemrc +## Optional: copy screenshots and database from the live website -gem install bundler + scp screenshots.debian.net:debshots-screenshots.tar . + tar -C public/ -xvf debshots-screenshots.tar + scp screenshots.debian.net:debshots.sql . + psql debshots < debshots.sql -bundle install --deployment +## Switch to production environment -=== Copy screenshots and database from the former live website + export RAILS_ENV=production -scp screenshots.debian.net:debshots-screenshots.tar . +## Migrate database -tar -C public/ -xvf debshots-screenshots.tar + bundle exec rake db:migrate RAILS_ENV=production -scp screenshots.debian.net:debshots.sql . +In config/database.yml: production: user/password/host must be commented out -psql debshots < debshots.sql +## Pre-render the static assets -Switch to production environment: + bundle exec rake assets:precompile RAILS_ENV=production -export RAILS_ENV=production +## Test the application: -Migrate database: + bundle exec rails s -b 0.0.0.0 -e production -bundle exec rake db:migrate RAILS_ENV=production +Point your browser to http://...:3000/ and check the web site. -(config/database.yml: production: user/password/host must be commented out) +## Create a user for moderation: -Convert screenshots to new format (paperclip): +Start a Rails console: -bundle exec rake debshots:screenshots_to_paperclip RAILS_ENV=production + bundle exec rails c -e production -Remove old screenshots directory structure: +Create a new user record: -rm -r public/live + user = User.create(realname: 'Christoph Haas', password: 'foobartest', email: 'email@christoph-haas.de') + user.save! -Render the assets: +## Tidy up the screenshot data -bundle exec rake assets:precompile RAILS_ENV=production + bundle exec rake debshots:remove_broken_screenshots + bundle exec rake debshots:remove_duplicate_images -Test the application: +## Import new package information -bundle exec rails s -b 0.0.0.0 -e production + bundle exec rake debshots:list_deb_repos + bundle exec rake debshots:update_from_deb_repos + bundle exec debshots:update_longdescription_from_deb_repos -http://...:3000/ -Create a user for moderation: -bundle exec rails c -e production +# Prepare production environment -user = User.create(realname: 'Christoph Haas', password: 'foobartest', email: 'email@christoph-haas.de') -user.save! +## Install passenger to run the application behind nginx -Tidy up the screenshot data: +Use the Ansible playbook to install Phusion Passenger - the component +that makes the actual Ruby on Rails web application accessible from +the Nginx web server. -bundle exec rake debshots:remove_broken_screenshots +## Example nginx vhost config -bundle exec rake debshots:remove_duplicate_images + more_clear_headers 'X-Runtime'; + more_clear_headers 'X-Powered-By'; + more_clear_headers 'Server'; + server_tokens off; -Import new package information: + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off; -bundle exec rake debshots:list_deb_repos + log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $remote_addr $request_uri "$http_referer" "$http_user_agent" $scheme'; -bundle exec rake debshots:update_from_deb_repos + # Performance + passenger_max_pool_size 8; + passenger_max_request_queue_size 200; -bundle exec debshots:update_longdescription_from_deb_repos + limit_req_zone $binary_remote_addr zone=one:20m rate=5r/s; -=== Install passenger to run the application behind nginx + # Frontend caching and static asset delivery + server { + listen 85.25.83.22:80 default_server; + listen 85.25.83.22:443 ssl; + ssl_certificate /etc/letsencrypt/live/screenshots.debian.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/screenshots.debian.net/privkey.pem; -https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/jessie/install_passenger.html + root /home/debshots/debshots/public; + server_name _; + access_log /var/log/nginx/cache-access.log proxy; -Example nginx vhost config: + add_header X-Cache-Status $upstream_cache_status; -## -# You should look at the following URL's in order to grasp a solid understanding -# of Nginx configuration files in order to fully unleash the power of Nginx. -# http://wiki.nginx.org/Pitfalls -# http://wiki.nginx.org/QuickStart -# http://wiki.nginx.org/Configuration -# -# Generally, you will want to move this file somewhere, and start with a clean -# file but keep this around for reference. Or just disable in sites-enabled. -# -# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. -## + # Tell the backend if the protocol used was HTTPS. Otherwise you get an infinite + # redirection loop because the backend assumes that HTTP was spoken behind the + # proxy. + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -#proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=assets:768m use_temp_path=off; + # Pass on the actual HTTP_HOST ("Host:" header) so that Rails can build proper absolute URLs + proxy_set_header Host $host; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + gzip_proxied any; -#add_header X-Cache-Status $upstream_cache_status; -#add_header X-Runtime 42; -more_clear_headers 'X-Runtime'; -more_clear_headers 'X-Powered-By'; -more_clear_headers 'Server'; -server_tokens off; + location / { + try_files /maintenance.html @backend; + add_header X-Coffee front-main; + } -#passenger_show_version_in_header off; + location /json/ { + try_files /maintenance.html @backend; + proxy_cache my_cache; + proxy_pass http://127.0.0.1:8888/; + proxy_cache_lock on; + proxy_cache_use_stale updating; + add_header X-Coffee front-json; + } -#log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $proxy_add_x_forwarded_for $request_uri "$http_referer" "$http_user_agent"'; -#access_log /var/log/nginx/debshots-access.log proxy; + location /favicon.ico { + alias /home/debshots/debshots/public/favicon.ico; + } -# Default server configuration -# -#passenger_log_level 5; + location @backend { + proxy_cache my_cache; + proxy_pass http://127.0.0.1:8888; -server { - listen 85.25.83.22:80 default_server; - #listen 80 default_server; - #listen [::]:80 default_server; + # Avoid cache stampede - rather update once and deliver stale content + proxy_cache_lock on; + proxy_cache_use_stale updating; + add_header X-Coffee front-rails; + } - # SSL configuration - # - # listen 443 ssl default_server; - # listen [::]:443 ssl default_server; - # - # Self signed certs generated by the ssl-cert package - # Don't use them in a production server! - # - # include snippets/snakeoil.conf; - - passenger_enabled on; - passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; - #passenger_pass_header X-Accel-Redirect; - - #proxy_cache_methods GET HEAD; - - root /home/debshots/debshots/public; - - # Add index.php to the list if you are using PHP - #index index.html index.htm index.nginx-debian.html; - - server_name _; - #proxy_cache assets; - #add_header X-Cache-Status $upstream_cache_status; - #proxy_ignore_headers Cache-Control; - - #add_header X-Debian rocks; - - location /assets/ { - alias /home/debshots/debshots/public/assets/; - expires 1h; - #include debshots_params; - add_header X-Coffee assets; - add_header Cache-Control public; - } - location /logo/ { - alias /home/debshots/debshots/public/logo/; - expires 1h; - add_header X-Coffee logo; - add_header Cache-Control public; - } - location /screenshots/ { - alias /home/debshots/debshots/public/screenshots/; - expires max; - add_header X-Coffee screenshots; - add_header Cache-Control public; - } - location /images/ { - alias /home/debshots/debshots/public/images/; - expires 1h; - add_header X-Coffee images; - add_header Cache-Control public; + location /assets/ { + alias /home/debshots/debshots/public/assets/; + expires 1h; + add_header X-Coffee front-assets; + } + location /logo/ { + alias /home/debshots/debshots/public/logo/; + expires 1h; + add_header X-Coffee front-logo; + } + location /screenshots/ { + alias /home/debshots/debshots/public/screenshots/; + expires max; + add_header X-Coffee front-screenshots; + } + location /images/ { + alias /home/debshots/debshots/public/images/; + expires 1h; + add_header X-Coffee front-images; + } } - # Send thumbnails using X-Sendfile / X-Accel-Redirect - # The correct thumbnail is computed by the Rails application so it cannot be served directly. - location /thumbnail/ { - expires 1d; - add_header X-Coffee thumbnail; - add_header Cache-Control public; + # Backend rails application + server { + listen 127.0.0.1:8888; - passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; - passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; - passenger_pass_header X-Accel-Redirect; + root /home/debshots/debshots/public; + + server_name _; + access_log /var/log/nginx/rails-access.log; + + passenger_enabled on; + passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; + + + # Send thumbnails using X-Sendfile / X-Accel-Redirect + # The correct thumbnail is computed by the Rails application so it cannot be served directly. + location /thumbnail/ { + expires 1h; + proxy_cache_valid 404 15m; + add_header X-Coffee back-thumbnail; + limit_req zone=one burst=30; + } + + # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) + location /public/ { + expires 1h; + add_header X-Coffee back-public; + + passenger_set_header X-Sendfile-Type X-Accel-Redirect; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + location /__send_file_accel/ { + internal; + alias /home/debshots/debshots/public/; + add_header X-Coffee back-accel; + expires 1d; + more_clear_headers 'Set-Cookie'; + } + + location /secretstatus { + stub_status; + access_log off; + allow all; + add_header X-Coffee back-status; + } } - # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) - location /public/ { - expires 1h; - add_header X-Coffee public; - add_header Cache-Control public; - passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; - passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; - passenger_pass_header X-Accel-Redirect; - } +# Supported URL paths (aka routes) - location /__send_file_accel/ { - internal; - alias /home/debshots/debshots/public/; - } -} - - -== Supported URL paths (aka routes) +_For up-to-date information please check config/routes._ /screenshots/:package_inital/:package/:(id)_:size.png * package = name of the package @@ -341,12 +368,12 @@ server { * Users: 403 * Moderators: re-approve a screenshot that was requested for deletion -== Cron jobs +## Cron jobs Parse Nginx access logs to count the number of visits of each package: -RAILS_ENV=production bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] + RAILS_ENV=production bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] As a cron job: -41 9 * * * cd $HOME/debshots ; RAILS_ENV=production /home/debshots/.rbenv/shims/bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] + 41 9 * * * cd $HOME/debshots ; RAILS_ENV=production /home/debshots/.rbenv/shims/bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1]