diff --git a/.gitignore b/.gitignore index f74f886..fa17ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,8 @@ /public/assets/ /public/system/ /public/screenshots/ +/public/screenshot/ +/public/cache/screenshot/ /public/live/ /storage diff --git a/Gemfile b/Gemfile index b018104..8530d89 100644 --- a/Gemfile +++ b/Gemfile @@ -6,9 +6,13 @@ gem 'rails', '~> 6.0' gem 'bootsnap' #gem 'webpacker' -# To create variants (different sizes) of screenshot images saved in ActiveStorage +# File handling +gem "shrine", "~> 3.0" + +# To create variants (different sizes) of screenshot images gem 'image_processing' gem 'mini_magick' +gem 'fastimage' # Use Puma as the app server gem 'puma', '~> 3.0' @@ -133,7 +137,3 @@ gem 'omniauth-amazon' gem 'omniauth-google-oauth2' gem 'omniauth-github' gem 'omniauth-openid' - -# Validations for ActiveStorage (which replaced Paperclip) are -# not yet available in Rails 6.0. So this is a third-party gem. -gem 'active_storage_validations' diff --git a/Gemfile.lock b/Gemfile.lock index 031b533..c742c56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -43,8 +43,6 @@ GEM erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - active_storage_validations (0.8.9) - rails (>= 5.2.0) activejob (6.0.3.4) activesupport (= 6.0.3.4) globalid (>= 0.3.6) @@ -99,6 +97,7 @@ GEM climate_control (0.2.0) coderay (1.1.3) concurrent-ruby (1.1.7) + content_disposition (1.0.0) cookies_eu (1.7.6) js_cookie_rails (~> 2.2.0) crass (1.0.6) @@ -109,11 +108,14 @@ GEM railties (>= 4.1.0) responders warden (~> 1.2.3) + down (5.2.0) + addressable (~> 2.5) e2mmap (0.1.0) erubi (1.9.0) execjs (2.7.0) faraday (1.0.1) multipart-post (>= 1.2, < 3) + fastimage (2.2.0) ffi (1.13.1) font-awesome-rails (4.7.0.5) railties (>= 3.2, < 6.1) @@ -330,6 +332,9 @@ GEM childprocess (>= 0.5, < 4.0) rubyzip (>= 1.2.2) shellany (0.0.1) + shrine (3.3.0) + content_disposition (~> 1.0) + down (~> 5.1) slim (4.1.0) temple (>= 0.7.6, < 0.9) tilt (>= 2.0.6, < 2.1) @@ -405,7 +410,6 @@ PLATFORMS ruby DEPENDENCIES - active_storage_validations better_errors binding_of_caller bootsnap @@ -414,6 +418,7 @@ DEPENDENCIES capybara cookies_eu devise + fastimage font-awesome-rails foundation-rails (~> 6.6) guard-minitest @@ -440,6 +445,7 @@ DEPENDENCIES sass-rails (~> 5.0) sdoc (~> 0.4.0) selenium-webdriver + shrine (~> 3.0) slim-rails solargraph spring diff --git a/app/models/screenshot.rb b/app/models/screenshot.rb index 927b95c..2f4de99 100644 --- a/app/models/screenshot.rb +++ b/app/models/screenshot.rb @@ -2,38 +2,21 @@ class Screenshot < ApplicationRecord belongs_to :package, inverse_of: :screenshots belongs_to :user, inverse_of: :screenshots - # Use paperclip gem to handle image files related to screenshots - # After migration from Paperclip to ActiveStorage: - has_one_attached :image - - # Validate using 'active_storage_validations' gem - validates :image, attached: true, content_type: { in: 'image/png', message: 'is not a valid PNG image file' } - - # Before migration from Paperclip to ActiveStorage: - # has_attached_file :image, - # styles: { :large => '800x600>', :thumb => '160x120>' }, - # default_url: '/images/dummy/no-screenshots-available.svg', - # path: ':rails_root/public/screenshots/:id_partition/:style.png', - # url: '/screenshots/:id_partition/:style.png' + # Paperclip + has_attached_file :image, + styles: { :large => '800x600>', :thumb => '160x120>' }, + default_url: '/images/dummy/no-screenshots-available.svg', + path: ':rails_root/public/screenshots/:id_partition/:style.png', + url: '/screenshots/:id_partition/:style.png' # validates_attachment_content_type :image, :content_type => 'image/png' # validates_with AttachmentSizeValidator, :attributes => :image, :less_than => 5.megabytes - # # validate :validate_image_is_unique + # validate :validate_image_is_unique # validates :delete_reason, length: { in: 5..100 }, allow_nil: true - # Validator that checks if the image has already been uploaded. - # A generic uniqueness validator does not work because it would attribute - # the error to the :image_fingerprint field and not the actual :image field. - # Currently this check makes sure that the same screenshot is not uploaded - # twice for the same package. However it allows the screenshot to be uploaded - # for two different packages. Restricting that further may happen later. - def validate_image_is_unique - # Look for images with the same checksum / image_fingerprint - if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id) - unless screenshot.id == self.id - errors.add(:image, "has already been uploaded for this package") - end - end - end + # Shrine + # include PaperclipShrineSynchronization # needs to be after `has_attached_file` + include ImageUploader::Attachment(:simage) # adds an `simage` virtual attribute + # Calculate how many days ago this screenshot has been uploaded def age_days @@ -180,4 +163,32 @@ class Screenshot < ApplicationRecord ).processed end end + + private + + # Validator that checks if the image has already been uploaded. + # A generic uniqueness validator does not work because it would attribute + # the error to the :image_fingerprint field and not the actual :image field. + # Currently this check makes sure that the same screenshot is not uploaded + # twice for the same package. However it allows the screenshot to be uploaded + # for two different packages. Restricting that further may happen later. + def validate_image_is_unique + # Look for images with the same checksum / image_fingerprint + if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id) + unless screenshot.id == self.id + errors.add(:image, "has already been uploaded for this package") + end + end + end + + # Check that the uploaded file can be handled by ImageMagick and is not broken + # def image_intact? + # begin + # #MiniMagick::Image.open(self.image.blob.open) + # MiniMagick::Image.read(self.image.download) + # rescue MiniMagick::Invalid => exc + # error.add(:image, 'appears to be a broken image file') + # end + # end + end diff --git a/app/uploaders/image_uploader.rb b/app/uploaders/image_uploader.rb new file mode 100644 index 0000000..4ae1cdb --- /dev/null +++ b/app/uploaders/image_uploader.rb @@ -0,0 +1,49 @@ +# This is a subclass of Shrine base that will be further configured for it's requirements. +# This will be included in the model to manage the file. + +class ImageUploader < Shrine + ALLOWED_TYPES = %w[image/jpeg image/png image/webp] + MAX_SIZE = 5*1024*1024 # 5 MB + MAX_DIMENSIONS = [3000, 3000] # 3000x3000 + + plugin :remove_attachment + plugin :pretty_location + plugin :validation_helpers + plugin :store_dimensions, log_subscriber: nil + # plugin :derivation_endpoint, prefix: "derivations/image" + + # File validations (requires `validation_helpers` plugin) + Attacher.validate do + validate_size 0..MAX_SIZE + + if validate_mime_type ALLOWED_TYPES + validate_max_dimensions MAX_DIMENSIONS + end + end + + # Thumbnails processor (requires `derivatives` plugin) + Attacher.derivatives do |original| + puts "original=#{original}" + magick = ImageProcessing::MiniMagick.source(original) + { + small: magick.resize_to_limit!(160, 120), + medium: magick.resize_to_limit!(800, 600), + large: magick.composite!('public/logo/watermark.png', gravity: 'east') + } + # GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb + + # THUMBNAILS.transform_values do |(width, height)| + # GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb + # end + end + + # Default to dynamic thumbnail URL (requires `default_url` plugin) + Attacher.default_url do |derivative: nil, **| + file&.derivation_url(:thumbnail, *THUMBNAILS.fetch(derivative)) if derivative + end + + # Dynamic thumbnail definition (requires `derivation_endpoint` plugin) + # derivation :thumbnail do |file, width, height| + # GenerateThumbnail.call(file, width.to_i, height.to_i) # lib/generate_thumbnail.rb + # end +end diff --git a/app/views/packages/_grid_thumbnail.slim b/app/views/packages/_grid_thumbnail.slim index 1565c32..dd9281d 100644 --- a/app/views/packages/_grid_thumbnail.slim +++ b/app/views/packages/_grid_thumbnail.slim @@ -4,7 +4,8 @@ a.black href=package_path(name: pkg.name) - if pkg.screenshots_approved.any? // TODO: smarter selection of the most useful screenshot instead of taking the first one - screenshot = pkg.screenshots.first - = image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail') + /= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail') + = image_tag(screenshot.simage_url(:small), alt: screenshot.caption, class: 'thumbnail') - else img.screenshot.thumbnail src="/images/dummy/no-screenshots-upload-one.svg" div diff --git a/app/views/packages/_medium_image.slim b/app/views/packages/_medium_image.slim new file mode 100644 index 0000000..22e2292 --- /dev/null +++ b/app/views/packages/_medium_image.slim @@ -0,0 +1,14 @@ +/ Show image in medium size with zoom option and caption below +a href =url_for(screenshot.simage_url(:large)) rel='fancybox-thumb' title=screenshot.caption data-fancybox='gallery' data-caption=screenshot_caption(screenshot) + .image-with-zoom-icon + = image_tag(url_for(screenshot.simage_url(:medium)), alt: screenshot.caption, class: 'thumbnail') + + .magnifying-glass-icon + i.fa.fa-search + +.imgcaption + = screenshot.description + +/ Display management buttons for admins only +- if user_signed_in? and current_user.is_admin? + = render(partial: 'admin_dropdown', locals: {screenshot: screenshot}) diff --git a/config/initializers/new_framework_defaults_5_2.rb b/config/initializers/new_framework_defaults_5_2.rb deleted file mode 100644 index c383d07..0000000 --- a/config/initializers/new_framework_defaults_5_2.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Be sure to restart your server when you modify this file. -# -# This file contains migration options to ease your Rails 5.2 upgrade. -# -# Once upgraded flip defaults one by one to migrate to the new default. -# -# Read the Guide for Upgrading Ruby on Rails for more info on each option. - -# Make Active Record use stable #cache_key alongside new #cache_version method. -# This is needed for recyclable cache keys. -# Rails.application.config.active_record.cache_versioning = true - -# Use AES-256-GCM authenticated encryption for encrypted cookies. -# Also, embed cookie expiry in signed or encrypted cookies for increased security. -# -# This option is not backwards compatible with earlier Rails versions. -# It's best enabled when your entire app is migrated and stable on 5.2. -# -# Existing cookies will be converted on read then written with the new scheme. -# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true - -# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages -# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true. -# Rails.application.config.active_support.use_authenticated_message_encryption = true - -# Add default protection from forgery to ActionController::Base instead of in -# ApplicationController. -# Rails.application.config.action_controller.default_protect_from_forgery = true - -# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and -# 'f' after migrating old data. -# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true - -# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header. -# Rails.application.config.active_support.use_sha1_digests = true - -# Make `form_with` generate id attributes for any generated HTML tags. -# Rails.application.config.action_view.form_with_generates_ids = true diff --git a/config/initializers/shrine.rb b/config/initializers/shrine.rb new file mode 100644 index 0000000..a932230 --- /dev/null +++ b/config/initializers/shrine.rb @@ -0,0 +1,21 @@ +# See also: https://github.com/erikdahlstrand/shrine-rails-example/blob/master/config/initializers/shrine.rb + +require "shrine" +require "shrine/storage/file_system" + +# both `cache` and `store` storages are needed +Shrine.storages = { + # Saves to ./public/screenshot/ID/image/… + cache: Shrine::Storage::FileSystem.new("public", prefix: "cache"), + store: Shrine::Storage::FileSystem.new("public", prefix: "/"), +} + +# See plugin documentation at https://shrinerb.com/docs/plugins/activerecord +Shrine.plugin :activerecord +Shrine.plugin :instrumentation +Shrine.plugin :determine_mime_type, analyzer: :marcel, log_subscriber: nil +Shrine.plugin :cached_attachment_data +Shrine.plugin :restore_cached_data +Shrine.plugin :derivatives # up front processing +Shrine.plugin :derivation_endpoint, # on-the-fly processing + secret_key: Rails.application.secret_key_base diff --git a/db/migrate/20201001000000_convert_to_active_storage.rb b/db/migrate/20201001000000_convert_to_active_storage.rb deleted file mode 100644 index d50f95d..0000000 --- a/db/migrate/20201001000000_convert_to_active_storage.rb +++ /dev/null @@ -1,105 +0,0 @@ -# Convert the screenshots from the deprecated Paperclip gem schema -# to the new ActiveStorage schema built into Rails 5.2. - -Dir[Rails.root.join("app/models/**/*.rb")].sort.each { |file| require file } - -class ConvertToActiveStorage < ActiveRecord::Migration[5.2] - require 'open-uri' - - def up - puts "Running up..." - # postgres - get_blob_id = 'LASTVAL()' - # mysql / mariadb - # get_blob_id = 'LAST_INSERT_ID()' - # sqlite - # get_blob_id = 'LAST_INSERT_ROWID()' - - puts "Preparing queries" - - active_storage_blob_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL) - INSERT INTO active_storage_blobs ( - key, filename, content_type, metadata, byte_size, checksum, created_at - ) VALUES ($1, $2, $3, '{}', $4, $5, $6) - SQL - - active_storage_attachment_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL) - INSERT INTO active_storage_attachments ( - name, record_type, record_id, blob_id, created_at - ) VALUES ($1, $2, $3, #{get_blob_id}, $4) - SQL - - - Rails.application.eager_load! - models = ActiveRecord::Base.descendants.reject(&:abstract_class?) - - transaction do - models.each do |model| - attachments = model.column_names.map do |c| - if c =~ /(.+)_file_name$/ - $1 - end - end.compact - - if attachments.blank? - next - end - - model.find_each.each do |instance| - puts "Model: #{instance}" - attachments.each do |attachment| - puts "Attachment: #{attachment}" - if instance.send(attachment).path.blank? - next - end - - ActiveRecord::Base.connection.raw_connection.exec_prepared( - 'active_storage_blob_statement', [ - key(instance, attachment), - instance.send("#{attachment}_file_name"), - instance.send("#{attachment}_content_type"), - instance.send("#{attachment}_file_size"), - checksum(instance.send(attachment)), - instance.updated_at.iso8601 - ]) - - ActiveRecord::Base.connection.raw_connection.exec_prepared( - 'active_storage_attachment_statement', [ - attachment, - model.name, - instance.id, - instance.updated_at.iso8601, - ]) - end - end - end - end - end - - def down - raise ActiveRecord::IrreversibleMigration - end - - private - - def key(instance, attachment) - SecureRandom.uuid - # Alternatively: - # filename = instance.send("#{attachment}_file_name") - # klass = instance.class.table_name - # id = instance.id - # id_partition = ("%09d".freeze % id).scan(/\d{3}/).join("/".freeze) - - # "#{klass}/#{attachment.pluralize}/#{id_partition}/original/#{filename}" - end - - def checksum(attachment) - # local files stored on disk: - url = attachment.path - Digest::MD5.base64digest(File.read(url)) - - # remote files stored on another person's computer: - # url = attachment.url - # Digest::MD5.base64digest(Net::HTTP.get(URI(url))) - end -end diff --git a/db/migrate/20201031112024_add_shrine_image_data_to_screenshot.rb b/db/migrate/20201031112024_add_shrine_image_data_to_screenshot.rb new file mode 100644 index 0000000..6458b71 --- /dev/null +++ b/db/migrate/20201031112024_add_shrine_image_data_to_screenshot.rb @@ -0,0 +1,5 @@ +class AddShrineImageDataToScreenshot < ActiveRecord::Migration[6.0] + def change + add_column :screenshots, :simage_data, :text + end +end diff --git a/db/schema.rb b/db/schema.rb index 5ae36b1..4b79335 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_10_01_000000) do +ActiveRecord::Schema.define(version: 2020_10_31_112024) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -99,6 +99,7 @@ ActiveRecord::Schema.define(version: 2020_10_01_000000) do t.datetime "image_updated_at" t.string "image_fingerprint" t.integer "user_id", default: 0 + t.text "image_data" t.index ["id", "approved"], name: "id_approved" t.index ["id", "uploaderhash"], name: "id_uploaderhash" end diff --git a/lib/tasks/paperclip_to_shrine.rake b/lib/tasks/paperclip_to_shrine.rake new file mode 100644 index 0000000..14144a3 --- /dev/null +++ b/lib/tasks/paperclip_to_shrine.rake @@ -0,0 +1,27 @@ +#require 'paperclip_shrine_synchronization' + +namespace :debshots do + desc "Convert images from Paperclip (deprecated) to Shrine" + + task paperclip_to_shrine: :environment do + + Screenshot.find_each do |screenshot| + puts "Screenshot… #{screenshot.id}" + path = screenshot.image.path + puts "Image path… #{path}" + + # Skip run if there is a Shrine attachment already (idempotency) + next if screenshot.simage + + # Upload file as Shrine + screenshot.simage = File.open(path) + + puts "Creating derivatives…" + screenshot.simage_derivatives! + + screenshot.save! + puts "------------" + end + + end +end