From 459e86e25a6b4135f824409981672b56692711c9 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Tue, 7 Aug 2018 18:36:53 +0200 Subject: [PATCH] Various fixes that prevented production deployment --- Gemfile | 4 +- ansible/debshots.yml | 64 +++++------ ansible/files/nginx.conf | 90 ++++++++------- app/views/logs/_paginator.slim | 6 +- app/views/packages/_paginator.slim | 6 +- app/views/welcome/home.slim | 69 ++++++------ config/environments/production.rb | 2 +- doc/README.Installation | 172 +++-------------------------- 8 files changed, 134 insertions(+), 279 deletions(-) diff --git a/Gemfile b/Gemfile index ee77e1f..9bfe6c1 100644 --- a/Gemfile +++ b/Gemfile @@ -83,12 +83,12 @@ gem 'will_paginate' # Style the paginator properly to use Zurb Foundation's style # https://github.com/acrogenesis/will_paginate-foundation # (Does not support Foundation 6 yet.) -#gem 'will_paginate-foundation' +gem 'will_paginate-foundation' # Lightbox image viewer for full-sized images gem 'fancybox-rails' -# Trying pg_search for full-text search… too slow with ranking… screw PostgreSQL +# Full-text search in PostgreSQL gem 'pg_search' # Use SLIM as our templating language diff --git a/ansible/debshots.yml b/ansible/debshots.yml index a377108..dcb6faa 100644 --- a/ansible/debshots.yml +++ b/ansible/debshots.yml @@ -1,38 +1,41 @@ --- - hosts: all + vars: ruby_version: 2.4.1 + handlers: - name: restart nginx service: name=nginx state=restarted + tasks: - name: Install packages - apt: name="{{item}}" state=installed + apt: name="{{item}}" state=present with_items: - htop - python-psycopg2 - ruby - #- passenger - #- nginx-full + - sudo + - nginx-extras - apt-transport-https - ca-certificates - #- build-essential - #- ruby-dev - libssl-dev - libreadline-dev - zlib1g-dev - libbz2-dev - libpq-dev - #- libcurl4-gnutls-dev - #- ruby-bundler - #- nodejs - git + - imagemagick + + - name: Remove conflicting packages + apt: name=python-openssl state=absent + - name: Enable Jessie backports repository apt_repository: repo="deb http://ftp.debian.org/debian jessie-backports main" - name: Set up american locales to avoid warnings about missing locales locale_gen: name=en_US.UTF-8 state=present - name: Install PostgreSQL - apt: name=postgresql state=installed + apt: name=postgresql state=present - name: Create PostgreSQL user become: yes become_user: postgres @@ -43,24 +46,6 @@ postgresql_db: name=debshots owner=debshots encoding=UTF-8 template=template0 - name: Create application user user: name=debshots groups=adm home=/opt/debshots shell=/bin/bash - #- name: Install Passenger Gem - # gem: name=passenger state=present - #- name: Install Passenger and Nginx - # shell: passenger-install-nginx-module --auto - #- name: Validate installation - # shell: passenger-config validate-install --auto - - apt_key: keyserver=keyserver.ubuntu.com id=561F9B9CAC40B2F7 - - apt_repository: repo="deb https://oss-binaries.phusionpassenger.com/apt/passenger jessie main" - - apt: name="{{item}}" state=installed - with_items: - - nginx-extras - - passenger - - name: Enable Passenger in main nginx.conf - lineinfile: - dest: /etc/nginx/nginx.conf - regexp: '# include /etc/nginx/passenger\.conf' - line: 'include /etc/nginx/passenger.conf;' - backrefs: yes # Install newer Ruby version using rbenv - name: Get rbenv @@ -80,17 +65,35 @@ src: files/debshots-bash-profile dest: /opt/debshots/.profile owner: debshots + - name: Build Ruby version - shell: sudo -iu debshots rbenv install -v {{ruby_version}} creates=/opt/debshots/.rbenv/versions/{{ruby_version}} + shell: rbenv install -v {{ruby_version}} creates=/opt/debshots/.rbenv/versions/{{ruby_version}} + become: yes + become_user: debshots + become_flags: "-i" + + - name: Set Ruby version to version {{ruby_version}} + shell: rbenv local {{ruby_version}} + become: yes + become_user: debshots + become_flags: "-i" - name: Install certbot to get certificates from Lets Encrypt apt: name=certbot default_release=jessie-backports - name: Install Bundler - gem: name=bundler + shell: gem install bundler + become: yes + become_user: debshots + become_flags: "-i" - name: Install dependencies - shell: sudo -iu debshots bundle + shell: bundle install + args: + chdir: /opt/debshots + become: yes + become_user: debshots + become_flags: "-i" - name: Make sure that the public/ directory is readable by nginx file: path=/opt/debshots/public mode='go+rx' @@ -105,4 +108,3 @@ restart nginx # - debug: msg="Please run: certbot certonly --webroot -w /opt/debshots/public -d screenshots.debian.net" - diff --git a/ansible/files/nginx.conf b/ansible/files/nginx.conf index 30e54cf..10645e3 100644 --- a/ansible/files/nginx.conf +++ b/ansible/files/nginx.conf @@ -24,10 +24,6 @@ proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $remote_addr $request_uri "$http_referer" "$http_user_agent" $scheme'; -# Performance -passenger_max_pool_size 8; -passenger_max_request_queue_size 200; - limit_req_zone $binary_remote_addr zone=one:20m rate=5r/s; # Frontend caching and static asset delivery @@ -37,7 +33,7 @@ server { ssl_certificate /etc/letsencrypt/live/screenshots.debian.net/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/screenshots.debian.net/privkey.pem; - root /opt/debshots/public; + #root /opt/debshots/public; server_name _; access_log /var/log/nginx/cache-access.log proxy; @@ -62,7 +58,7 @@ server { location /json/ { try_files /maintenance.html @backend; proxy_cache my_cache; - proxy_pass http://127.0.0.1:8888/; + proxy_pass http://127.0.0.1:3000/; proxy_cache_lock on; proxy_cache_use_stale updating; add_header X-Coffee front-json; @@ -74,12 +70,17 @@ server { location @backend { proxy_cache my_cache; - proxy_pass http://127.0.0.1:8888; + proxy_set_header x_debian_sso_dn $ssl_client_s_dn; + proxy_pass http://127.0.0.1:3000; # Avoid cache stampede - rather update once and deliver stale content proxy_cache_lock on; proxy_cache_use_stale updating; add_header X-Coffee front-rails; + + # Use sendfile mechanism to deliver files directly + proxy_set_header X-Sendfile-Type X-Accel-Redirect; + proxy_set_header X-Accel-Mapping /opt/debshots/public/=/__send_file_accel/; } location /assets/ { @@ -105,49 +106,46 @@ server { } # Backend rails application -server { - listen 127.0.0.1:8888; +# server { +# listen 127.0.0.1:3000; - root /opt/debshots/public; +# root /opt/debshots/public; - server_name _; - access_log /var/log/nginx/rails-access.log; +# server_name _; +# access_log /var/log/nginx/rails-access.log; - passenger_enabled on; - passenger_ruby /opt/debshots/.rbenv/versions/2.4.1/bin/ruby; +# # Send thumbnails using X-Sendfile / X-Accel-Redirect +# # The correct thumbnail is computed by the Rails application so it cannot be served directly. +# location /thumbnail/ { +# expires 1h; +# proxy_cache_valid 404 15m; +# add_header X-Coffee back-thumbnail; +# limit_req zone=one burst=30; +# } +# # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) +# location /public/ { +# expires 1h; +# add_header X-Coffee back-public; - # Send thumbnails using X-Sendfile / X-Accel-Redirect - # The correct thumbnail is computed by the Rails application so it cannot be served directly. - location /thumbnail/ { - expires 1h; - proxy_cache_valid 404 15m; - add_header X-Coffee back-thumbnail; - limit_req zone=one burst=30; - } +# # passenger_set_header X-Sendfile-Type X-Accel-Redirect; +# # passenger_env_var HTTP_X_ACCEL_MAPPING /opt/debshots/public/=/__send_file_accel/; +# # passenger_pass_header X-Accel-Redirect; +# } - # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) - location /public/ { - expires 1h; - add_header X-Coffee back-public; +# # Deliver static files directly from Nginx. See https://mattbrictson.com/accelerated-rails-downloads +# location /__send_file_accel/ { +# internal; +# alias /opt/debshots/public/; +# add_header X-Coffee back-accel; +# expires 1d; +# more_clear_headers 'Set-Cookie'; +# } - passenger_set_header X-Sendfile-Type X-Accel-Redirect; - passenger_env_var HTTP_X_ACCEL_MAPPING /opt/debshots/public/=/__send_file_accel/; - passenger_pass_header X-Accel-Redirect; - } - - location /__send_file_accel/ { - internal; - alias /opt/debshots/public/; - add_header X-Coffee back-accel; - expires 1d; - more_clear_headers 'Set-Cookie'; - } - - location /secretstatus { - stub_status; - access_log off; - allow all; - add_header X-Coffee back-status; - } -} +# location /secretstatus { +# stub_status; +# access_log off; +# allow all; +# add_header X-Coffee back-status; +# } +# } diff --git a/app/views/logs/_paginator.slim b/app/views/logs/_paginator.slim index 81eab7d..6418f78 100644 --- a/app/views/logs/_paginator.slim +++ b/app/views/logs/_paginator.slim @@ -1,8 +1,8 @@ - if @logs.length>0 // Use different pagination navigators depending on the screen width div.show-for-large - =will_paginate @logs, :renderer => FoundationPaginationRenderer, :inner_window => 3 + =will_paginate @logs, :renderer => FoundationPagination::Rails, :inner_window => 3 div.show-for-medium-only - =will_paginate @logs, :renderer => FoundationPaginationRenderer, :inner_window => 1 + =will_paginate @logs, :renderer => FoundationPagination::Rails, :inner_window => 1 div.show-for-small-only - =will_paginate @logs, :renderer => FoundationPaginationRenderer, :page_links => false + =will_paginate @logs, :renderer => FoundationPagination::Rails, :page_links => false diff --git a/app/views/packages/_paginator.slim b/app/views/packages/_paginator.slim index 59f82bb..a80d5af 100644 --- a/app/views/packages/_paginator.slim +++ b/app/views/packages/_paginator.slim @@ -1,8 +1,8 @@ - if @packages.length>0 // Use different pagination navigators depending on the screen width div.show-for-large - =will_paginate @packages, :renderer => FoundationPaginationRenderer, :inner_window => 3 + =will_paginate @packages, :renderer => FoundationPagination::Rails, :inner_window => 3 div.show-for-medium-only - =will_paginate @packages, :renderer => FoundationPaginationRenderer, :inner_window => 1 + =will_paginate @packages, :renderer => FoundationPagination::Rails, :inner_window => 1 div.show-for-small-only - =will_paginate @packages, :renderer => FoundationPaginationRenderer, :page_links => false + =will_paginate @packages, :renderer => FoundationPagination::Rails, :page_links => false diff --git a/app/views/welcome/home.slim b/app/views/welcome/home.slim index 7386339..33df0ee 100644 --- a/app/views/welcome/home.slim +++ b/app/views/welcome/home.slim @@ -23,37 +23,38 @@ .input-group-button input.button.postfix type="submit" value="Search" -.grid-container - .grid-x.align-spaced - .large-3.medium-12.small-12.cell.text-center - h2 Newest upload - .thumb-height - a href=package_path(@newest_upload.package.name) - = image_tag(@newest_upload.image.url(:thumb, timestamp: false), alt: @newest_upload.caption) - p - strong - =@newest_upload.package.name - ' : - =@newest_upload.package.description - .large-3.medium-12.small-12.cell.text-center - h2 Most popular - .thumb-height - a href=package_path(@most_popular_package.name) - img src=@most_popular_package.screenshots.first.image.url(:thumb, timestamp: false) - p - strong - =@most_popular_package.name - ' : - =@most_popular_package.description - .large-3.medium-12.small-12.cell.text-center - h2 Contribute - .thumb-height - a href=upload_path(@most_wanted_package.name) - img src="/images/dummy/no-screenshots-upload-one.svg" width="160" - p - / TODO: Randomly choose one of ~30 packages - ' We lack a screenshot for the - strong =>@most_wanted_package.name - ' package. Can you - a href=package_path(@most_wanted_package.name) provide one - ' ? +- if @newest_upload + .grid-container + .grid-x.align-spaced + .large-3.medium-12.small-12.cell.text-center + h2 Newest upload + .thumb-height + a href=package_path(@newest_upload.package.name) + = image_tag(@newest_upload.image.url(:thumb, timestamp: false), alt: @newest_upload.caption) + p + strong + =@newest_upload.package.name + ' : + =@newest_upload.package.description + .large-3.medium-12.small-12.cell.text-center + h2 Most popular + .thumb-height + a href=package_path(@most_popular_package.name) + img src=@most_popular_package.screenshots.first.image.url(:thumb, timestamp: false) + p + strong + =@most_popular_package.name + ' : + =@most_popular_package.description + .large-3.medium-12.small-12.cell.text-center + h2 Contribute + .thumb-height + a href=upload_path(@most_wanted_package.name) + img src="/images/dummy/no-screenshots-upload-one.svg" width="160" + p + / TODO: Randomly choose one of ~30 packages + ' We lack a screenshot for the + strong =>@most_wanted_package.name + ' package. Can you + a href=package_path(@most_wanted_package.name) provide one + ' ? diff --git a/config/environments/production.rb b/config/environments/production.rb index 72d7f9d..a0fbc4f 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -43,7 +43,7 @@ Rails.application.configure do # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. - config.force_ssl = true + #config.force_ssl = true # Use the lowest log level to ensure availability of diagnostic information # when problems arise. diff --git a/doc/README.Installation b/doc/README.Installation index 12a1fc0..19e898a 100644 --- a/doc/README.Installation +++ b/doc/README.Installation @@ -5,15 +5,12 @@ Prerequisites: A Debian Jessie server. Install PIP to get a recent version of Ansible: apt install python-pip + pip install ansible -Tun the Ansible playbook that comes with the debshots repository: +Run the Ansible playbook that comes with the debshots repository: cd ansible - ansible-playbook debshost.yml - -_(If you get "command not found" then run "hash -r" or "rehash" in your -shell and try again.)_ - + ansible-playbook debshots.yml # Installing debshots the manual way @@ -80,17 +77,23 @@ Clone the debshots Git repository. ## Migrate database - bundle exec rake db:migrate RAILS_ENV=production + rails db:schema:load RAILS_ENV=production + - rails db:migrate RAILS_ENV=production - In config/database.yml: production: user/password/host must be commented out +## Fill/update the database with information from APT repositories + + rails debshots:update_from_deb_repos RAILS_ENV=production + rails debshots:update_longdescription_from_deb_repos RAILS_ENV=production + ## Pre-render the static assets - bundle exec rake assets:precompile RAILS_ENV=production - + rails assets:precompile RAILS_ENV=production + ## Test the application: - bundle exec rails s -b 0.0.0.0 -e production + rails s -b 127.0.0.1 -e production Point your browser to http://...:3000/ and check the web site. @@ -117,155 +120,6 @@ Create a new user record: bundle exec debshots:update_longdescription_from_deb_repos - -# Prepare production environment - -## Install passenger to run the application behind nginx - -Use the Ansible playbook to install Phusion Passenger - the component -that makes the actual Ruby on Rails web application accessible from -the Nginx web server. - -## Example nginx vhost config - - more_clear_headers 'X-Runtime'; - more_clear_headers 'X-Powered-By'; - more_clear_headers 'Server'; - server_tokens off; - - proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off; - - log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $remote_addr $request_uri "$http_referer" "$http_user_agent" $scheme'; - - # Performance - passenger_max_pool_size 8; - passenger_max_request_queue_size 200; - - limit_req_zone $binary_remote_addr zone=one:20m rate=5r/s; - - # Frontend caching and static asset delivery - server { - listen 85.25.83.22:80 default_server; - listen 85.25.83.22:443 ssl; - ssl_certificate /etc/letsencrypt/live/screenshots.debian.net/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/screenshots.debian.net/privkey.pem; - - root /home/debshots/debshots/public; - server_name _; - access_log /var/log/nginx/cache-access.log proxy; - - add_header X-Cache-Status $upstream_cache_status; - - # Tell the backend if the protocol used was HTTPS. Otherwise you get an infinite - # redirection loop because the backend assumes that HTTP was spoken behind the - # proxy. - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # Pass on the actual HTTP_HOST ("Host:" header) so that Rails can build proper absolute URLs - proxy_set_header Host $host; - gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - gzip_proxied any; - - location / { - try_files /maintenance.html @backend; - add_header X-Coffee front-main; - } - - location /json/ { - try_files /maintenance.html @backend; - proxy_cache my_cache; - proxy_pass http://127.0.0.1:8888/; - proxy_cache_lock on; - proxy_cache_use_stale updating; - add_header X-Coffee front-json; - } - - location /favicon.ico { - alias /home/debshots/debshots/public/favicon.ico; - } - - location @backend { - proxy_cache my_cache; - proxy_pass http://127.0.0.1:8888; - - # Avoid cache stampede - rather update once and deliver stale content - proxy_cache_lock on; - proxy_cache_use_stale updating; - add_header X-Coffee front-rails; - } - - location /assets/ { - alias /home/debshots/debshots/public/assets/; - expires 1h; - add_header X-Coffee front-assets; - } - location /logo/ { - alias /home/debshots/debshots/public/logo/; - expires 1h; - add_header X-Coffee front-logo; - } - location /screenshots/ { - alias /home/debshots/debshots/public/screenshots/; - expires max; - add_header X-Coffee front-screenshots; - } - location /images/ { - alias /home/debshots/debshots/public/images/; - expires 1h; - add_header X-Coffee front-images; - } - } - - # Backend rails application - server { - listen 127.0.0.1:8888; - - root /home/debshots/debshots/public; - - server_name _; - access_log /var/log/nginx/rails-access.log; - - passenger_enabled on; - passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; - - - # Send thumbnails using X-Sendfile / X-Accel-Redirect - # The correct thumbnail is computed by the Rails application so it cannot be served directly. - location /thumbnail/ { - expires 1h; - proxy_cache_valid 404 15m; - add_header X-Coffee back-thumbnail; - limit_req zone=one burst=30; - } - - # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) - location /public/ { - expires 1h; - add_header X-Coffee back-public; - - passenger_set_header X-Sendfile-Type X-Accel-Redirect; - passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; - passenger_pass_header X-Accel-Redirect; - } - - location /__send_file_accel/ { - internal; - alias /home/debshots/debshots/public/; - add_header X-Coffee back-accel; - expires 1d; - more_clear_headers 'Set-Cookie'; - } - - location /secretstatus { - stub_status; - access_log off; - allow all; - add_header X-Coffee back-status; - } - } - - # Supported URL paths (aka routes) _For up-to-date information please check config/routes._