Migration to Shrine attachment handling

.gitignore

@@ -18,6 +18,8 @@
 /public/assets/
 /public/system/
 /public/screenshots/
+/public/screenshot/
+/public/cache/screenshot/
 /public/live/
 /storage
 

Gemfile

@@ -6,9 +6,13 @@ gem 'rails', '~> 6.0'
 gem 'bootsnap'
 #gem 'webpacker'
 
-# To create variants (different sizes) of screenshot images saved in ActiveStorage
+# File handling
+gem "shrine", "~> 3.0"
+
+# To create variants (different sizes) of screenshot images
 gem 'image_processing'
 gem 'mini_magick'
+gem 'fastimage'
 
 # Use Puma as the app server
 gem 'puma', '~> 3.0'
@@ -133,7 +137,3 @@ gem 'omniauth-amazon'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-github'
 gem 'omniauth-openid'
-
-# Validations for ActiveStorage (which replaced Paperclip) are
-# not yet available in Rails 6.0. So this is a third-party gem.
-gem 'active_storage_validations'

Gemfile.lock

@@ -43,8 +43,6 @@ GEM
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    active_storage_validations (0.8.9)
-      rails (>= 5.2.0)
     activejob (6.0.3.4)
       activesupport (= 6.0.3.4)
       globalid (>= 0.3.6)
@@ -99,6 +97,7 @@ GEM
     climate_control (0.2.0)
     coderay (1.1.3)
     concurrent-ruby (1.1.7)
+    content_disposition (1.0.0)
     cookies_eu (1.7.6)
       js_cookie_rails (~> 2.2.0)
     crass (1.0.6)
@@ -109,11 +108,14 @@ GEM
       railties (>= 4.1.0)
       responders
       warden (~> 1.2.3)
+    down (5.2.0)
+      addressable (~> 2.5)
     e2mmap (0.1.0)
     erubi (1.9.0)
     execjs (2.7.0)
     faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
+    fastimage (2.2.0)
     ffi (1.13.1)
     font-awesome-rails (4.7.0.5)
       railties (>= 3.2, < 6.1)
@@ -330,6 +332,9 @@ GEM
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
     shellany (0.0.1)
+    shrine (3.3.0)
+      content_disposition (~> 1.0)
+      down (~> 5.1)
     slim (4.1.0)
       temple (>= 0.7.6, < 0.9)
       tilt (>= 2.0.6, < 2.1)
@@ -405,7 +410,6 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  active_storage_validations
   better_errors
   binding_of_caller
   bootsnap
@@ -414,6 +418,7 @@ DEPENDENCIES
   capybara
   cookies_eu
   devise
+  fastimage
   font-awesome-rails
   foundation-rails (~> 6.6)
   guard-minitest
@@ -440,6 +445,7 @@ DEPENDENCIES
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   selenium-webdriver
+  shrine (~> 3.0)
   slim-rails
   solargraph
   spring

app/models/screenshot.rb

@@ -2,38 +2,21 @@ class Screenshot < ApplicationRecord
   belongs_to :package, inverse_of: :screenshots
   belongs_to :user, inverse_of: :screenshots
 
-  # Use paperclip gem to handle image files related to screenshots
+  # Paperclip
-  # After migration from Paperclip to ActiveStorage:
+  has_attached_file :image,
-  has_one_attached :image
+    styles: { :large => '800x600>', :thumb => '160x120>' },
-
+    default_url: '/images/dummy/no-screenshots-available.svg',
-  # Validate using 'active_storage_validations' gem
+    path: ':rails_root/public/screenshots/:id_partition/:style.png',
-  validates :image, attached: true, content_type: { in: 'image/png', message: 'is not a valid PNG image file' }
+    url: '/screenshots/:id_partition/:style.png'
-
-  # Before migration from Paperclip to ActiveStorage:
-  # has_attached_file :image,
-  #   styles: { :large => '800x600>', :thumb => '160x120>' },
-  #   default_url: '/images/dummy/no-screenshots-available.svg',
-  #   path: ':rails_root/public/screenshots/:id_partition/:style.png',
-  #   url: '/screenshots/:id_partition/:style.png'
   # validates_attachment_content_type :image, :content_type => 'image/png'
   # validates_with AttachmentSizeValidator, :attributes => :image, :less_than => 5.megabytes
-  # #  validate :validate_image_is_unique
+  #  validate :validate_image_is_unique
   # validates :delete_reason, length: { in: 5..100 }, allow_nil: true
 
-  # Validator that checks if the image has already been uploaded.
+  # Shrine
-  # A generic uniqueness validator does not work because it would attribute
+ # include PaperclipShrineSynchronization # needs to be after `has_attached_file`
-  # the error to the :image_fingerprint field and not the actual :image field.
+  include ImageUploader::Attachment(:simage) # adds an `simage` virtual attribute
-  # Currently this check makes sure that the same screenshot is not uploaded
+
-  # twice for the same package. However it allows the screenshot to be uploaded
-  # for two different packages. Restricting that further may happen later.
-  def validate_image_is_unique
-    # Look for images with the same checksum / image_fingerprint
-    if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
-      unless screenshot.id == self.id
-        errors.add(:image, "has already been uploaded for this package")
-      end
-    end
-  end
 
   # Calculate how many days ago this screenshot has been uploaded
   def age_days
@@ -180,4 +163,32 @@ class Screenshot < ApplicationRecord
       ).processed
     end
   end
+
+  private
+
+  # Validator that checks if the image has already been uploaded.
+  # A generic uniqueness validator does not work because it would attribute
+  # the error to the :image_fingerprint field and not the actual :image field.
+  # Currently this check makes sure that the same screenshot is not uploaded
+  # twice for the same package. However it allows the screenshot to be uploaded
+  # for two different packages. Restricting that further may happen later.
+  def validate_image_is_unique
+    # Look for images with the same checksum / image_fingerprint
+    if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
+      unless screenshot.id == self.id
+        errors.add(:image, "has already been uploaded for this package")
+      end
+    end
+  end
+
+  # Check that the uploaded file can be handled by ImageMagick and is not broken
+  # def image_intact?
+  #   begin
+  #     #MiniMagick::Image.open(self.image.blob.open)
+  #     MiniMagick::Image.read(self.image.download)      
+  #   rescue MiniMagick::Invalid => exc
+  #     error.add(:image, 'appears to be a broken image file')
+  #   end      
+  # end
+
 end

app/uploaders/image_uploader.rb

@@ -0,0 +1,49 @@
+# This is a subclass of Shrine base that will be further configured for it's requirements.
+# This will be included in the model to manage the file.
+
+class ImageUploader < Shrine
+  ALLOWED_TYPES  = %w[image/jpeg image/png image/webp]
+  MAX_SIZE       = 5*1024*1024 # 5 MB
+  MAX_DIMENSIONS = [3000, 3000] # 3000x3000
+
+  plugin :remove_attachment
+  plugin :pretty_location
+  plugin :validation_helpers
+  plugin :store_dimensions, log_subscriber: nil
+  # plugin :derivation_endpoint, prefix: "derivations/image"
+
+  # File validations (requires `validation_helpers` plugin)
+  Attacher.validate do
+    validate_size 0..MAX_SIZE
+
+    if validate_mime_type ALLOWED_TYPES
+      validate_max_dimensions MAX_DIMENSIONS
+    end
+  end
+
+  # Thumbnails processor (requires `derivatives` plugin)
+  Attacher.derivatives do |original|
+    puts "original=#{original}"
+    magick = ImageProcessing::MiniMagick.source(original)
+    {
+      small: magick.resize_to_limit!(160, 120),
+      medium: magick.resize_to_limit!(800, 600),
+      large: magick.composite!('public/logo/watermark.png', gravity: 'east')
+    }
+    # GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
+
+    # THUMBNAILS.transform_values do |(width, height)|
+    #   GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
+    # end
+  end
+
+  # Default to dynamic thumbnail URL (requires `default_url` plugin)
+  Attacher.default_url do |derivative: nil, **|
+    file&.derivation_url(:thumbnail, *THUMBNAILS.fetch(derivative)) if derivative
+  end
+
+  # Dynamic thumbnail definition (requires `derivation_endpoint` plugin)
+  # derivation :thumbnail do |file, width, height|
+  #   GenerateThumbnail.call(file, width.to_i, height.to_i) # lib/generate_thumbnail.rb
+  # end
+end

app/views/packages/_grid_thumbnail.slim

@@ -4,7 +4,8 @@ a.black href=package_path(name: pkg.name)
     - if pkg.screenshots_approved.any?
       // TODO: smarter selection of the most useful screenshot instead of taking the first one
       - screenshot = pkg.screenshots.first
-      = image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
+      /= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
+      = image_tag(screenshot.simage_url(:small), alt: screenshot.caption, class: 'thumbnail')
     - else
       img.screenshot.thumbnail src="/images/dummy/no-screenshots-upload-one.svg"
   div

app/views/packages/_medium_image.slim

@@ -0,0 +1,14 @@
+/ Show image in medium size with zoom option and caption below
+a href =url_for(screenshot.simage_url(:large)) rel='fancybox-thumb' title=screenshot.caption data-fancybox='gallery' data-caption=screenshot_caption(screenshot)
+  .image-with-zoom-icon
+    = image_tag(url_for(screenshot.simage_url(:medium)), alt: screenshot.caption, class: 'thumbnail')
+
+    .magnifying-glass-icon
+      i.fa.fa-search
+
+.imgcaption
+  = screenshot.description
+
+/ Display management buttons for admins only
+- if user_signed_in? and current_user.is_admin?
+  = render(partial: 'admin_dropdown', locals: {screenshot: screenshot})

config/initializers/new_framework_defaults_5_2.rb

@@ -1,38 +0,0 @@
-# Be sure to restart your server when you modify this file.
-#
-# This file contains migration options to ease your Rails 5.2 upgrade.
-#
-# Once upgraded flip defaults one by one to migrate to the new default.
-#
-# Read the Guide for Upgrading Ruby on Rails for more info on each option.
-
-# Make Active Record use stable #cache_key alongside new #cache_version method.
-# This is needed for recyclable cache keys.
-# Rails.application.config.active_record.cache_versioning = true
-
-# Use AES-256-GCM authenticated encryption for encrypted cookies.
-# Also, embed cookie expiry in signed or encrypted cookies for increased security.
-#
-# This option is not backwards compatible with earlier Rails versions.
-# It's best enabled when your entire app is migrated and stable on 5.2.
-#
-# Existing cookies will be converted on read then written with the new scheme.
-# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true
-
-# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages
-# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true.
-# Rails.application.config.active_support.use_authenticated_message_encryption = true
-
-# Add default protection from forgery to ActionController::Base instead of in
-# ApplicationController.
-# Rails.application.config.action_controller.default_protect_from_forgery = true
-
-# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and
-# 'f' after migrating old data.
-# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
-
-# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header.
-# Rails.application.config.active_support.use_sha1_digests = true
-
-# Make `form_with` generate id attributes for any generated HTML tags.
-# Rails.application.config.action_view.form_with_generates_ids = true

config/initializers/shrine.rb

@@ -0,0 +1,21 @@
+# See also: https://github.com/erikdahlstrand/shrine-rails-example/blob/master/config/initializers/shrine.rb
+
+require "shrine"
+require "shrine/storage/file_system"
+
+# both `cache` and `store` storages are needed
+Shrine.storages = {
+  # Saves to ./public/screenshot/ID/image/…
+  cache: Shrine::Storage::FileSystem.new("public", prefix: "cache"),
+  store: Shrine::Storage::FileSystem.new("public", prefix: "/"),
+}
+
+# See plugin documentation at https://shrinerb.com/docs/plugins/activerecord
+Shrine.plugin :activerecord
+Shrine.plugin :instrumentation
+Shrine.plugin :determine_mime_type, analyzer: :marcel, log_subscriber: nil
+Shrine.plugin :cached_attachment_data
+Shrine.plugin :restore_cached_data
+Shrine.plugin :derivatives          # up front processing
+Shrine.plugin :derivation_endpoint, # on-the-fly processing
+  secret_key: Rails.application.secret_key_base

db/migrate/20201001000000_convert_to_active_storage.rb

@@ -1,105 +0,0 @@
-# Convert the screenshots from the deprecated Paperclip gem schema
-# to the new ActiveStorage schema built into Rails 5.2.
-
-Dir[Rails.root.join("app/models/**/*.rb")].sort.each { |file| require file }
-
-class ConvertToActiveStorage < ActiveRecord::Migration[5.2]
-  require 'open-uri'
-
-  def up
-    puts "Running up..."
-    # postgres
-    get_blob_id = 'LASTVAL()'
-    # mysql / mariadb
-    # get_blob_id = 'LAST_INSERT_ID()'
-    # sqlite
-    # get_blob_id = 'LAST_INSERT_ROWID()'
-
-    puts "Preparing queries"
-
-    active_storage_blob_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL)
-      INSERT INTO active_storage_blobs (
-        key, filename, content_type, metadata, byte_size, checksum, created_at
-      ) VALUES ($1, $2, $3, '{}', $4, $5, $6)
-    SQL
-
-    active_storage_attachment_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL)
-      INSERT INTO active_storage_attachments (
-        name, record_type, record_id, blob_id, created_at
-      ) VALUES ($1, $2, $3, #{get_blob_id}, $4)
-    SQL
-
-
-    Rails.application.eager_load!
-    models = ActiveRecord::Base.descendants.reject(&:abstract_class?)
-
-    transaction do
-      models.each do |model|
-        attachments = model.column_names.map do |c|
-          if c =~ /(.+)_file_name$/
-            $1
-          end
-        end.compact
-
-        if attachments.blank?
-          next
-        end
-
-        model.find_each.each do |instance|
-          puts "Model: #{instance}"
-          attachments.each do |attachment|
-            puts "Attachment: #{attachment}"
-            if instance.send(attachment).path.blank?
-              next
-            end
-
-            ActiveRecord::Base.connection.raw_connection.exec_prepared(
-              'active_storage_blob_statement', [
-                key(instance, attachment),
-                instance.send("#{attachment}_file_name"),
-                instance.send("#{attachment}_content_type"),
-                instance.send("#{attachment}_file_size"),
-                checksum(instance.send(attachment)),
-                instance.updated_at.iso8601
-              ])
-
-            ActiveRecord::Base.connection.raw_connection.exec_prepared(
-              'active_storage_attachment_statement', [
-                attachment,
-                model.name,
-                instance.id,
-                instance.updated_at.iso8601,
-              ])
-          end
-        end
-      end
-    end
-  end
-
-  def down
-    raise ActiveRecord::IrreversibleMigration
-  end
-
-  private
-
-  def key(instance, attachment)
-    SecureRandom.uuid
-    # Alternatively:
-    # filename = instance.send("#{attachment}_file_name")
-    # klass = instance.class.table_name
-    # id = instance.id
-    # id_partition = ("%09d".freeze % id).scan(/\d{3}/).join("/".freeze)
-
-    # "#{klass}/#{attachment.pluralize}/#{id_partition}/original/#{filename}"
-  end
-
-  def checksum(attachment)
-    # local files stored on disk:
-    url = attachment.path
-    Digest::MD5.base64digest(File.read(url))
-
-    # remote files stored on another person's computer:
-    # url = attachment.url
-    # Digest::MD5.base64digest(Net::HTTP.get(URI(url)))
-  end
-end

db/migrate/20201031112024_add_shrine_image_data_to_screenshot.rb

@@ -0,0 +1,5 @@
+class AddShrineImageDataToScreenshot < ActiveRecord::Migration[6.0]
+  def change
+    add_column :screenshots, :simage_data, :text
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_10_01_000000) do
+ActiveRecord::Schema.define(version: 2020_10_31_112024) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -99,6 +99,7 @@ ActiveRecord::Schema.define(version: 2020_10_01_000000) do
     t.datetime "image_updated_at"
     t.string "image_fingerprint"
     t.integer "user_id", default: 0
+    t.text "image_data"
     t.index ["id", "approved"], name: "id_approved"
     t.index ["id", "uploaderhash"], name: "id_uploaderhash"
   end

lib/tasks/paperclip_to_shrine.rake

@@ -0,0 +1,27 @@
+#require 'paperclip_shrine_synchronization'
+
+namespace :debshots do
+  desc "Convert images from Paperclip (deprecated) to Shrine"
+
+  task paperclip_to_shrine: :environment do
+
+    Screenshot.find_each do |screenshot|
+      puts "Screenshot… #{screenshot.id}"
+      path = screenshot.image.path
+      puts "Image path… #{path}"
+
+      # Skip run if there is a Shrine attachment already (idempotency)
+      next if screenshot.simage
+
+      # Upload file as Shrine
+      screenshot.simage = File.open(path)
+
+      puts "Creating derivatives…"
+      screenshot.simage_derivatives!
+
+      screenshot.save!
+      puts "------------"
+    end
+
+  end
+end