From dd91412671cce6053c6f738de64dadfe6e5a5f46 Mon Sep 17 00:00:00 2001 From: Christoph Haas Date: Wed, 19 Apr 2017 13:08:02 +0200 Subject: [PATCH] Documented installation and setup procedure --- doc/README.md | 352 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 doc/README.md diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..b011d34 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,352 @@ +== Installing debshots + +=== Prepare PostgreSQL database server and user (ident-based authentication) + +apt install postgresql + +su - postgres + +createuser -d debshots + +createdb -O debshots -E UTF8 -T template0 debshots + +=== Create an application user + +adduser debshots + +Allow the user to parse access log files from Nginx: + +adduser debshots adm + +su - debshots + +=== Install the required Ruby version using rbenv + +https://github.com/rbenv/rbenv + +git clone https://github.com/rbenv/rbenv.git ~/.rbenv + +echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bash_profile + + +# Load rbenv automatically by appending +# the following to ~/.bashrc: + +eval "$(rbenv init -)" + + +https://github.com/rbenv/ruby-build#readme + +git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build + +=== Get the application + +Clone the debshots Git repository. + +=== Install the Ruby dependencies + +rbenv install + +echo "gem: --no-rdoc --no-ri" > ~/.gemrc + +gem install bundler + +bundle install --deployment + +=== Copy screenshots and database from the former live website + +scp screenshots.debian.net:debshots-screenshots.tar . + +tar -C public/ -xvf debshots-screenshots.tar + +scp screenshots.debian.net:debshots.sql . + +psql debshots < debshots.sql + +Switch to production environment: + +export RAILS_ENV=production + +Migrate database: + +bundle exec rake db:migrate RAILS_ENV=production + +(config/database.yml: production: user/password/host must be commented out) + +Convert screenshots to new format (paperclip): + +bundle exec rake debshots:screenshots_to_paperclip RAILS_ENV=production + +Remove old screenshots directory structure: + +rm -r public/live + +Render the assets: + +bundle exec rake assets:precompile RAILS_ENV=production + +Test the application: + +bundle exec rails s -b 0.0.0.0 -e production + +http://...:3000/ + +Create a user for moderation: + +bundle exec rails c -e production + +user = User.create(realname: 'Christoph Haas', password: 'foobartest', email: 'email@christoph-haas.de') +user.save! + +Tidy up the screenshot data: + +bundle exec rake debshots:remove_broken_screenshots + +bundle exec rake debshots:remove_duplicate_images + +Import new package information: + +bundle exec rake debshots:list_deb_repos + +bundle exec rake debshots:update_from_deb_repos + +bundle exec debshots:update_longdescription_from_deb_repos + +=== Install passenger to run the application behind nginx + +https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/jessie/install_passenger.html + +Example nginx vhost config: + +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# http://wiki.nginx.org/Pitfalls +# http://wiki.nginx.org/QuickStart +# http://wiki.nginx.org/Configuration +# +# Generally, you will want to move this file somewhere, and start with a clean +# file but keep this around for reference. Or just disable in sites-enabled. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +#proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=assets:768m use_temp_path=off; + +#add_header X-Cache-Status $upstream_cache_status; +#add_header X-Runtime 42; +more_clear_headers 'X-Runtime'; +more_clear_headers 'X-Powered-By'; +more_clear_headers 'Server'; +server_tokens off; + +#passenger_show_version_in_header off; + +#log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $proxy_add_x_forwarded_for $request_uri "$http_referer" "$http_user_agent"'; +#access_log /var/log/nginx/debshots-access.log proxy; + +# Default server configuration +# +#passenger_log_level 5; + +server { + listen 85.25.83.22:80 default_server; + #listen 80 default_server; + #listen [::]:80 default_server; + + # SSL configuration + # + # listen 443 ssl default_server; + # listen [::]:443 ssl default_server; + # + # Self signed certs generated by the ssl-cert package + # Don't use them in a production server! + # + # include snippets/snakeoil.conf; + + passenger_enabled on; + passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; + #passenger_pass_header X-Accel-Redirect; + + #proxy_cache_methods GET HEAD; + + root /home/debshots/debshots/public; + + # Add index.php to the list if you are using PHP + #index index.html index.htm index.nginx-debian.html; + + server_name _; + #proxy_cache assets; + #add_header X-Cache-Status $upstream_cache_status; + #proxy_ignore_headers Cache-Control; + + #add_header X-Debian rocks; + + location /assets/ { + alias /home/debshots/debshots/public/assets/; + expires 1h; + #include debshots_params; + add_header X-Coffee assets; + add_header Cache-Control public; + } + location /logo/ { + alias /home/debshots/debshots/public/logo/; + expires 1h; + add_header X-Coffee logo; + add_header Cache-Control public; + } + location /screenshots/ { + alias /home/debshots/debshots/public/screenshots/; + expires max; + add_header X-Coffee screenshots; + add_header Cache-Control public; + } + location /images/ { + alias /home/debshots/debshots/public/images/; + expires 1h; + add_header X-Coffee images; + add_header Cache-Control public; + } + + # Send thumbnails using X-Sendfile / X-Accel-Redirect + # The correct thumbnail is computed by the Rails application so it cannot be served directly. + location /thumbnail/ { + expires 1d; + add_header X-Coffee thumbnail; + add_header Cache-Control public; + + passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) + location /public/ { + expires 1h; + add_header X-Coffee public; + add_header Cache-Control public; + + passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + location /__send_file_accel/ { + internal; + alias /home/debshots/debshots/public/; + } +} + + +== Supported URL paths (aka routes) + +/screenshots/:package_inital/:package/:(id)_:size.png +* package = name of the package +* id = numerical ID of the screenshot +* size = "large" or "small" +* Returns a PNG image of a screenshot +* Should be served as a static asset + +/image/:(id)_:size.png +* id = numerical ID of the screenshot +* size = "large" or "small" +* Returns a PNG image of a screenshot +* Usually used for unapproved images that should just be shown to the uploader + +/about +* Return an informational HTML page + +/packages +* Return a list of packages with pagination + +/packages/with_screenshots +* Return a list of packages with pagination +* Select only those packages that have screenshots. + +/packages/without_screenshots +* Return a list of packages with pagination +* Select only those packages that do not have screenshots. + +/packages/games_without_screenshots (deprecated) +* Return a list of packages with pagination +* Select only those packages that belong to Debian's "games" section + +/json/packages +* Return a JSON data structure of all packages + +/json/screenshots +* Return a JSON data structure containing packages and screenshots + +/json/packages-without-screenshots +* Return a JSON data structure of those packages that do not have any screenshots + +/packages/moderate +* Return a HTML page showing packages that need moderation +* Must only be available to logged in moderators + +/upload +* Return a HTML page containing an upload form for new screenshots + +/guidelines (deprecated) +* Return a HTML page showing general guidelines for creating screenshots +* Should just redirect to the /upload page + +/upload/:package +* ?? + +/uploadfile +* ?? + +/login +* Show a login form + +/logout +* Logout the user by invalidating the cookie session. + +/package/:package +* Show details of a certain package + +/json/package/:package +* Return a JSON data structure containing details of a certain package + +/thumbnail/:package +/thumbnail-404/:package (deprecated) +* Return a small (160x120 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code + +/thumbnail-with-version/:package/:version +* Return a small (160x120 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code +* Prefer screenshots of a certain version + +/screenshot/:package +/screenshot-404/:package (deprecated) +* Return a large (320x240 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code + +/screenshot-with-version/:package/:version +* Return a large (320x240 pixel) PNG screenshot file of a package +* If the package does not have any screenshots then return a 404 code +* Prefer screenshots of a certain version + +/delete_screenshot/:screenshot +* Users: request deletion of a screenshot +* Moderators: delete a screenshot + +/approve_screenshot/:screenshot +* Users: 403 +* Moderators: approve a screenshot + +/keep_screenshot/:screenshot +* Users: 403 +* Moderators: re-approve a screenshot that was requested for deletion + +== Cron jobs + +Parse Nginx access logs to count the number of visits of each package: + +RAILS_ENV=production bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1] + +As a cron job: + +41 9 * * * cd $HOME/debshots ; RAILS_ENV=production /home/debshots/.rbenv/shims/bundle exec rake debshots:accesslog2visits[/var/log/nginx/cache-access.log.1]