diff --git a/README.Developer b/README.Developer deleted file mode 100644 index 0b55df7..0000000 --- a/README.Developer +++ /dev/null @@ -1,128 +0,0 @@ -# Full text search - -The pg_search Ruby gem is used for full-text searched in Rails. -See: https://github.com/Casecommons/pg_search - -Reformatted SQL query for fulltext search: - -SELECT COUNT(*) FROM "packages" WHERE ( - ( - ( - to_tsvector('simple', coalesce("packages"."name"::text, '')) - || - to_tsvector('simple', coalesce("packages"."description"::text, '')) - || - to_tsvector('simple', coalesce("packages"."long_description"::text, '')) - ) - @@ - ( - to_tsquery('simple', '''' || 'browser' || ' ''') - ) - ) -) - -# Index: - -CREATE INDEX packages_gin ON packages USING GIN( (to_tsvector('simple', -coalesce("packages"."name"::text, '')) || to_tsvector('simple', -coalesce("packages"."description"::text, '')) || to_tsvector('simple', -coalesce("packages"."long_description"::text, ''))) ); - ---------- - -Used: https://github.com/semaperepelitsa/jquery.fileupload-rails -Which is a gem for: https://github.com/blueimp/jQuery-File-Upload - -Paperclip for attachment/screenshot file handlind within ActiveRecord -https://github.com/thoughtbot/paperclip - --- - -Components: - -- Rails 5 (widespread, up-to-date, solves many common problems) -- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search) -- Git (most people nowadays use Git) -- jQuery (well known and easy to use) -- Zurb Foundation (integrates nicer than Twitter Bootstrap) -- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails) - --- - -Importer - -How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat - --- - -Logging from within the application into the database's "log" table: - -Log.log "foo" -or with section (default='frontend'): -Log.log "foo", "importer" - --- - -Future stuff ------------- - -A possible migration adding a unique constraint to avoid duplicate images - -class AddScreenshotUniqueConstraintImageFingerprint < ActiveRecord::Migration - def change - add_index :screenshots, [:package_id, :image_fingerprint], :unique => true - end -end - -A separate validation in the Screenshots model has been implemented already. -This unique constraint would raise an error and not trigger a validation error -properly unless the separate model validation is in place. - --- - -Deployment ----------- -To prepare your computer to run Debshots 2.x either for development -or production you need to… - -- unpack the project onto your disk somewhere -- install Ruby 2.0.0 via rvm (see http://rvm.io/) -- install the library header packages so that the required Ruby gems - can be compiled: - sudo apt-get install libbz2-dev libpq-dev libyaml-dev -- install a PostgreSQL database: - sudo apt-get install postgresql -- edit the PostgreSQL authentication configuration at - /etc/postgres/.../pg_hba.conf to allow password-based logins - from localhost. Change the line starting with *local* to read: - local all all md5 -- create a user and databases - - sudo su - postgres - - createuser -s -P debshots - - createdb debshots -- for development only: - - sudo su - postgres - - createdb debshots_dev -- migrate the data from the Debshots 1.x based production server: - - bin/get-data-from-old-live-server.sh -- install all Ruby gems required to run the application: - - gem install bundle - - bundler -- run database migrations for development - - RAILS_ENV=development rake db:migrate -- import long description information for the packages into development database - - RAILS_ENV=development rake debshots:update_longdescription_from_deb_repos -- run database migrations for production - - RAILS_ENV=production rake db:migrate -- RAILS_ENV=production rake debshots:update_longdescription_from_deb_repos - - import long description information for the packages into production database -- prepare the static assets (CSS, JS) for production - - RAILS_ENV=production rake assets:precompile -- run the production web server - - RAILS_ENV=production rails s - -To create admin users ---------------------- - -rails c -> User.new(email: 'email@christoph-haas.de', password: 'secret').save diff --git a/doc/README.Developer b/doc/README.Developer new file mode 100644 index 0000000..407991e --- /dev/null +++ b/doc/README.Developer @@ -0,0 +1,93 @@ +# Full text search + +The pg_search Ruby gem is used for full-text searched in Rails. +See: https://github.com/Casecommons/pg_search + +Reformatted SQL query for fulltext search: + +SELECT COUNT(*) FROM "packages" WHERE ( + ( + ( + to_tsvector('simple', coalesce("packages"."name"::text, '')) + || + to_tsvector('simple', coalesce("packages"."description"::text, '')) + || + to_tsvector('simple', coalesce("packages"."long_description"::text, '')) + ) + @@ + ( + to_tsquery('simple', '''' || 'browser' || ' ''') + ) + ) +) + +# Index: + +CREATE INDEX packages_gin ON packages USING GIN( (to_tsvector('simple', +coalesce("packages"."name"::text, '')) || to_tsvector('simple', +coalesce("packages"."description"::text, '')) || to_tsvector('simple', +coalesce("packages"."long_description"::text, ''))) ); + +--------- + +Used: https://github.com/semaperepelitsa/jquery.fileupload-rails +Which is a gem for: https://github.com/blueimp/jQuery-File-Upload + +Paperclip for attachment/screenshot file handlind within ActiveRecord +https://github.com/thoughtbot/paperclip + +-- + +Components: + +- Rails 5 (widespread, up-to-date, solves many common problems) +- PostgreSQL (fast, easy, I don't like MySQL, good fulltext search) +- Git (most people nowadays use Git) +- jQuery (well known and easy to use) +- Zurb Foundation (integrates nicer than Twitter Bootstrap) +- Guard for automated testing (http://www.rubydoc.info/gems/guard-rails) + +-- + +Importer + +How Debian repositories are organized: https://wiki.debian.org/RepositoryFormat + +-- + +Logging from within the application into the database's "log" table: + +Log.log "foo" +or with section (default='frontend'): +Log.log "foo", "importer" + +-- + +To create admin users +--------------------- + +rails c +> User.new(email: 'email@christoph-haas.de', password: 'secret').save + +--- + +Multiple distributions (e.g. Ubuntu, Debian) +- new table: distributions + - id + - name (e.g. "Debian") + - description (e.g. "The Debian GNU/Linux distribution") + - url (http://www.debian.org/) + - type (apt, later: rpm) + - logo + +- screenshots: new field for source_id + +Admin interface to manage sources + +--- + +AppStream data +- screenshots: new field for type ('user-generated', 'upstream', 'logo') + user-generated: uploaded by a random user + upstream: official screenshot from the developer + logo: not a screenshot - just a logo diff --git a/README.rdoc b/doc/README.rdoc similarity index 100% rename from README.rdoc rename to doc/README.rdoc diff --git a/doc/nginx/default b/doc/nginx/default new file mode 100644 index 0000000..a033186 --- /dev/null +++ b/doc/nginx/default @@ -0,0 +1,196 @@ +## +# You should look at the following URL's in order to grasp a solid understanding +# of Nginx configuration files in order to fully unleash the power of Nginx. +# http://wiki.nginx.org/Pitfalls +# http://wiki.nginx.org/QuickStart +# http://wiki.nginx.org/Configuration +# +# Generally, you will want to move this file somewhere, and start with a clean +# file but keep this around for reference. Or just disable in sites-enabled. +# +# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples. +## + +#proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=assets:768m use_temp_path=off; + +#add_header X-Cache-Status $upstream_cache_status; +#add_header X-Runtime 42; +more_clear_headers 'X-Runtime'; +more_clear_headers 'X-Powered-By'; +more_clear_headers 'Server'; +server_tokens off; + +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off; + +log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $proxy_add_x_forwarded_for $request_uri "$http_referer" "$http_user_agent"'; + +#passenger_show_version_in_header off; + +#log_format proxy '[$time_local] Cache: $upstream_cache_status $upstream_addr $upstream_response_time $status $bytes_sent $proxy_add_x_forwarded_for $request_uri "$http_referer" "$http_user_agent"'; + +# Default server configuration +# +#passenger_log_level 5; + +# Performance +passenger_max_pool_size 8; +passenger_max_request_queue_size 200; + +#limit_req_zone $binary_remote_addr zone=one:10m rate=5r/s; + +# Frontend caching and static asset delivery +server { + listen 85.25.83.22:80 default_server; + listen 85.25.83.22:443 ssl; + ssl_certificate ssl/screenshots.debian.net.crt; + ssl_certificate_key ssl/screenshots.debian.net.key; + + root /home/debshots/debshots/public; + server_name _; + access_log /var/log/nginx/cache-access.log proxy; + + add_header X-Cache-Status $upstream_cache_status; + + # Pass on the actual HTTP_HOST ("Host:" header) so that Rails can build proper absolute URLs + #proxy_set_header Host $http_host; + proxy_set_header Host $host; + #proxy_cache_key $scheme$proxy_host$request_uri; + # + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + gzip_proxied any; + + location / { + try_files /maintenance.html @backend; + add_header X-Coffee main; + } + + location /json/ { + try_files /maintenance.html @backend; + add_header X-Coffee json; + } + + location /favicon.ico { + # try_files /maintenance.html @backend; + alias /home/debshots/debshots/public/favicon.ico; + } + + location @backend { + proxy_cache my_cache; + proxy_pass http://127.0.0.1:8888; + + # Avoid cache stampede - rather update once and deliver stale content + proxy_cache_lock on; + proxy_cache_use_stale updating; + } + + location /assets/ { + alias /home/debshots/debshots/public/assets/; + expires 1h; + add_header X-Coffee assets; + } + location /logo/ { + alias /home/debshots/debshots/public/logo/; + expires 1h; + add_header X-Coffee logo; + } + location /screenshots/ { + alias /home/debshots/debshots/public/screenshots/; + expires max; + add_header X-Coffee screenshots; + } + location /images/ { + alias /home/debshots/debshots/public/images/; + expires 1h; + add_header X-Coffee images; + } +} + +# Backend rails application +server { + #listen 85.25.83.22:80 default_server; + #listen 85.25.83.22:443 ssl; + #ssl_certificate ssl/screenshots.debian.net.crt; + #ssl_certificate_key ssl/screenshots.debian.net.key; + listen 127.0.0.1:8888; + + root /home/debshots/debshots/public; + + server_name _; + access_log /var/log/nginx/rails-access.log; + + passenger_enabled on; + passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; + #expires 1d; + #limit_req zone=one burst=5; + + #location / { + # try_files /maintenance.html @passenger; + #} +# +# location @passenger { +# #passenger_enabled on; +# #rails_env production; +# passenger_enabled on; +# passenger_ruby /home/debshots/.rbenv/versions/2.3.1/bin/ruby; +# #limit_req zone=one burst=5; +# add_header X-Coffee app; +## } + + + # Send thumbnails using X-Sendfile / X-Accel-Redirect + # The correct thumbnail is computed by the Rails application so it cannot be served directly. + location /thumbnail/ { + expires 1h; + add_header X-Coffee thumbnail; + + passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + # Send public assets using X-Sendfile / X-Accel-Redirect (e.g. public/images/dummy/...) + location /public/ { + expires 1h; + add_header X-Coffee public; + + passenger_set_header X-Sendfile-Type "X-Accel-Redirect"; + passenger_env_var HTTP_X_ACCEL_MAPPING /home/debshots/debshots/public/=/__send_file_accel/; + passenger_pass_header X-Accel-Redirect; + } + + location /__send_file_accel/ { + internal; + alias /home/debshots/debshots/public/; + add_header X-Coffee accel; + expires 1d; + more_clear_headers 'Set-Cookie'; + } + + location /secretstatus { + stub_status; + access_log off; + allow all; + add_header X-Coffee status; + #deny all; + } +} + + +# Virtual Host configuration for example.com +# +# You can move that to a different file under sites-available/ and symlink that +# to sites-enabled/ to enable it. +# +#server { +# listen 80; +# listen [::]:80; +# +# server_name example.com; +# +# root /var/www/example.com; +# index index.html; +# +# location / { +# try_files $uri $uri/ =404; +# } +#}