Migration to Shrine attachment handling

This commit is contained in:
Christoph Haas 2020-11-01 15:07:16 +01:00
parent 2d53397dda
commit c4f5741b0c
13 changed files with 175 additions and 181 deletions

2
.gitignore vendored
View file

@ -18,6 +18,8 @@
/public/assets/
/public/system/
/public/screenshots/
/public/screenshot/
/public/cache/screenshot/
/public/live/
/storage

10
Gemfile
View file

@ -6,9 +6,13 @@ gem 'rails', '~> 6.0'
gem 'bootsnap'
#gem 'webpacker'
# To create variants (different sizes) of screenshot images saved in ActiveStorage
# File handling
gem "shrine", "~> 3.0"
# To create variants (different sizes) of screenshot images
gem 'image_processing'
gem 'mini_magick'
gem 'fastimage'
# Use Puma as the app server
gem 'puma', '~> 3.0'
@ -133,7 +137,3 @@ gem 'omniauth-amazon'
gem 'omniauth-google-oauth2'
gem 'omniauth-github'
gem 'omniauth-openid'
# Validations for ActiveStorage (which replaced Paperclip) are
# not yet available in Rails 6.0. So this is a third-party gem.
gem 'active_storage_validations'

View file

@ -43,8 +43,6 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
active_storage_validations (0.8.9)
rails (>= 5.2.0)
activejob (6.0.3.4)
activesupport (= 6.0.3.4)
globalid (>= 0.3.6)
@ -99,6 +97,7 @@ GEM
climate_control (0.2.0)
coderay (1.1.3)
concurrent-ruby (1.1.7)
content_disposition (1.0.0)
cookies_eu (1.7.6)
js_cookie_rails (~> 2.2.0)
crass (1.0.6)
@ -109,11 +108,14 @@ GEM
railties (>= 4.1.0)
responders
warden (~> 1.2.3)
down (5.2.0)
addressable (~> 2.5)
e2mmap (0.1.0)
erubi (1.9.0)
execjs (2.7.0)
faraday (1.0.1)
multipart-post (>= 1.2, < 3)
fastimage (2.2.0)
ffi (1.13.1)
font-awesome-rails (4.7.0.5)
railties (>= 3.2, < 6.1)
@ -330,6 +332,9 @@ GEM
childprocess (>= 0.5, < 4.0)
rubyzip (>= 1.2.2)
shellany (0.0.1)
shrine (3.3.0)
content_disposition (~> 1.0)
down (~> 5.1)
slim (4.1.0)
temple (>= 0.7.6, < 0.9)
tilt (>= 2.0.6, < 2.1)
@ -405,7 +410,6 @@ PLATFORMS
ruby
DEPENDENCIES
active_storage_validations
better_errors
binding_of_caller
bootsnap
@ -414,6 +418,7 @@ DEPENDENCIES
capybara
cookies_eu
devise
fastimage
font-awesome-rails
foundation-rails (~> 6.6)
guard-minitest
@ -440,6 +445,7 @@ DEPENDENCIES
sass-rails (~> 5.0)
sdoc (~> 0.4.0)
selenium-webdriver
shrine (~> 3.0)
slim-rails
solargraph
spring

View file

@ -2,38 +2,21 @@ class Screenshot < ApplicationRecord
belongs_to :package, inverse_of: :screenshots
belongs_to :user, inverse_of: :screenshots
# Use paperclip gem to handle image files related to screenshots
# After migration from Paperclip to ActiveStorage:
has_one_attached :image
# Validate using 'active_storage_validations' gem
validates :image, attached: true, content_type: { in: 'image/png', message: 'is not a valid PNG image file' }
# Before migration from Paperclip to ActiveStorage:
# has_attached_file :image,
# styles: { :large => '800x600>', :thumb => '160x120>' },
# default_url: '/images/dummy/no-screenshots-available.svg',
# path: ':rails_root/public/screenshots/:id_partition/:style.png',
# url: '/screenshots/:id_partition/:style.png'
# Paperclip
has_attached_file :image,
styles: { :large => '800x600>', :thumb => '160x120>' },
default_url: '/images/dummy/no-screenshots-available.svg',
path: ':rails_root/public/screenshots/:id_partition/:style.png',
url: '/screenshots/:id_partition/:style.png'
# validates_attachment_content_type :image, :content_type => 'image/png'
# validates_with AttachmentSizeValidator, :attributes => :image, :less_than => 5.megabytes
# # validate :validate_image_is_unique
# validate :validate_image_is_unique
# validates :delete_reason, length: { in: 5..100 }, allow_nil: true
# Validator that checks if the image has already been uploaded.
# A generic uniqueness validator does not work because it would attribute
# the error to the :image_fingerprint field and not the actual :image field.
# Currently this check makes sure that the same screenshot is not uploaded
# twice for the same package. However it allows the screenshot to be uploaded
# for two different packages. Restricting that further may happen later.
def validate_image_is_unique
# Look for images with the same checksum / image_fingerprint
if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
unless screenshot.id == self.id
errors.add(:image, "has already been uploaded for this package")
end
end
end
# Shrine
# include PaperclipShrineSynchronization # needs to be after `has_attached_file`
include ImageUploader::Attachment(:simage) # adds an `simage` virtual attribute
# Calculate how many days ago this screenshot has been uploaded
def age_days
@ -180,4 +163,32 @@ class Screenshot < ApplicationRecord
).processed
end
end
private
# Validator that checks if the image has already been uploaded.
# A generic uniqueness validator does not work because it would attribute
# the error to the :image_fingerprint field and not the actual :image field.
# Currently this check makes sure that the same screenshot is not uploaded
# twice for the same package. However it allows the screenshot to be uploaded
# for two different packages. Restricting that further may happen later.
def validate_image_is_unique
# Look for images with the same checksum / image_fingerprint
if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
unless screenshot.id == self.id
errors.add(:image, "has already been uploaded for this package")
end
end
end
# Check that the uploaded file can be handled by ImageMagick and is not broken
# def image_intact?
# begin
# #MiniMagick::Image.open(self.image.blob.open)
# MiniMagick::Image.read(self.image.download)
# rescue MiniMagick::Invalid => exc
# error.add(:image, 'appears to be a broken image file')
# end
# end
end

View file

@ -0,0 +1,49 @@
# This is a subclass of Shrine base that will be further configured for it's requirements.
# This will be included in the model to manage the file.
class ImageUploader < Shrine
ALLOWED_TYPES = %w[image/jpeg image/png image/webp]
MAX_SIZE = 5*1024*1024 # 5 MB
MAX_DIMENSIONS = [3000, 3000] # 3000x3000
plugin :remove_attachment
plugin :pretty_location
plugin :validation_helpers
plugin :store_dimensions, log_subscriber: nil
# plugin :derivation_endpoint, prefix: "derivations/image"
# File validations (requires `validation_helpers` plugin)
Attacher.validate do
validate_size 0..MAX_SIZE
if validate_mime_type ALLOWED_TYPES
validate_max_dimensions MAX_DIMENSIONS
end
end
# Thumbnails processor (requires `derivatives` plugin)
Attacher.derivatives do |original|
puts "original=#{original}"
magick = ImageProcessing::MiniMagick.source(original)
{
small: magick.resize_to_limit!(160, 120),
medium: magick.resize_to_limit!(800, 600),
large: magick.composite!('public/logo/watermark.png', gravity: 'east')
}
# GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
# THUMBNAILS.transform_values do |(width, height)|
# GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
# end
end
# Default to dynamic thumbnail URL (requires `default_url` plugin)
Attacher.default_url do |derivative: nil, **|
file&.derivation_url(:thumbnail, *THUMBNAILS.fetch(derivative)) if derivative
end
# Dynamic thumbnail definition (requires `derivation_endpoint` plugin)
# derivation :thumbnail do |file, width, height|
# GenerateThumbnail.call(file, width.to_i, height.to_i) # lib/generate_thumbnail.rb
# end
end

View file

@ -4,7 +4,8 @@ a.black href=package_path(name: pkg.name)
- if pkg.screenshots_approved.any?
// TODO: smarter selection of the most useful screenshot instead of taking the first one
- screenshot = pkg.screenshots.first
= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
/= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
= image_tag(screenshot.simage_url(:small), alt: screenshot.caption, class: 'thumbnail')
- else
img.screenshot.thumbnail src="/images/dummy/no-screenshots-upload-one.svg"
div

View file

@ -0,0 +1,14 @@
/ Show image in medium size with zoom option and caption below
a href =url_for(screenshot.simage_url(:large)) rel='fancybox-thumb' title=screenshot.caption data-fancybox='gallery' data-caption=screenshot_caption(screenshot)
.image-with-zoom-icon
= image_tag(url_for(screenshot.simage_url(:medium)), alt: screenshot.caption, class: 'thumbnail')
.magnifying-glass-icon
i.fa.fa-search
.imgcaption
= screenshot.description
/ Display management buttons for admins only
- if user_signed_in? and current_user.is_admin?
= render(partial: 'admin_dropdown', locals: {screenshot: screenshot})

View file

@ -1,38 +0,0 @@
# Be sure to restart your server when you modify this file.
#
# This file contains migration options to ease your Rails 5.2 upgrade.
#
# Once upgraded flip defaults one by one to migrate to the new default.
#
# Read the Guide for Upgrading Ruby on Rails for more info on each option.
# Make Active Record use stable #cache_key alongside new #cache_version method.
# This is needed for recyclable cache keys.
# Rails.application.config.active_record.cache_versioning = true
# Use AES-256-GCM authenticated encryption for encrypted cookies.
# Also, embed cookie expiry in signed or encrypted cookies for increased security.
#
# This option is not backwards compatible with earlier Rails versions.
# It's best enabled when your entire app is migrated and stable on 5.2.
#
# Existing cookies will be converted on read then written with the new scheme.
# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true
# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages
# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true.
# Rails.application.config.active_support.use_authenticated_message_encryption = true
# Add default protection from forgery to ActionController::Base instead of in
# ApplicationController.
# Rails.application.config.action_controller.default_protect_from_forgery = true
# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and
# 'f' after migrating old data.
# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header.
# Rails.application.config.active_support.use_sha1_digests = true
# Make `form_with` generate id attributes for any generated HTML tags.
# Rails.application.config.action_view.form_with_generates_ids = true

View file

@ -0,0 +1,21 @@
# See also: https://github.com/erikdahlstrand/shrine-rails-example/blob/master/config/initializers/shrine.rb
require "shrine"
require "shrine/storage/file_system"
# both `cache` and `store` storages are needed
Shrine.storages = {
# Saves to ./public/screenshot/ID/image/…
cache: Shrine::Storage::FileSystem.new("public", prefix: "cache"),
store: Shrine::Storage::FileSystem.new("public", prefix: "/"),
}
# See plugin documentation at https://shrinerb.com/docs/plugins/activerecord
Shrine.plugin :activerecord
Shrine.plugin :instrumentation
Shrine.plugin :determine_mime_type, analyzer: :marcel, log_subscriber: nil
Shrine.plugin :cached_attachment_data
Shrine.plugin :restore_cached_data
Shrine.plugin :derivatives # up front processing
Shrine.plugin :derivation_endpoint, # on-the-fly processing
secret_key: Rails.application.secret_key_base

View file

@ -1,105 +0,0 @@
# Convert the screenshots from the deprecated Paperclip gem schema
# to the new ActiveStorage schema built into Rails 5.2.
Dir[Rails.root.join("app/models/**/*.rb")].sort.each { |file| require file }
class ConvertToActiveStorage < ActiveRecord::Migration[5.2]
require 'open-uri'
def up
puts "Running up..."
# postgres
get_blob_id = 'LASTVAL()'
# mysql / mariadb
# get_blob_id = 'LAST_INSERT_ID()'
# sqlite
# get_blob_id = 'LAST_INSERT_ROWID()'
puts "Preparing queries"
active_storage_blob_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL)
INSERT INTO active_storage_blobs (
key, filename, content_type, metadata, byte_size, checksum, created_at
) VALUES ($1, $2, $3, '{}', $4, $5, $6)
SQL
active_storage_attachment_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL)
INSERT INTO active_storage_attachments (
name, record_type, record_id, blob_id, created_at
) VALUES ($1, $2, $3, #{get_blob_id}, $4)
SQL
Rails.application.eager_load!
models = ActiveRecord::Base.descendants.reject(&:abstract_class?)
transaction do
models.each do |model|
attachments = model.column_names.map do |c|
if c =~ /(.+)_file_name$/
$1
end
end.compact
if attachments.blank?
next
end
model.find_each.each do |instance|
puts "Model: #{instance}"
attachments.each do |attachment|
puts "Attachment: #{attachment}"
if instance.send(attachment).path.blank?
next
end
ActiveRecord::Base.connection.raw_connection.exec_prepared(
'active_storage_blob_statement', [
key(instance, attachment),
instance.send("#{attachment}_file_name"),
instance.send("#{attachment}_content_type"),
instance.send("#{attachment}_file_size"),
checksum(instance.send(attachment)),
instance.updated_at.iso8601
])
ActiveRecord::Base.connection.raw_connection.exec_prepared(
'active_storage_attachment_statement', [
attachment,
model.name,
instance.id,
instance.updated_at.iso8601,
])
end
end
end
end
end
def down
raise ActiveRecord::IrreversibleMigration
end
private
def key(instance, attachment)
SecureRandom.uuid
# Alternatively:
# filename = instance.send("#{attachment}_file_name")
# klass = instance.class.table_name
# id = instance.id
# id_partition = ("%09d".freeze % id).scan(/\d{3}/).join("/".freeze)
# "#{klass}/#{attachment.pluralize}/#{id_partition}/original/#{filename}"
end
def checksum(attachment)
# local files stored on disk:
url = attachment.path
Digest::MD5.base64digest(File.read(url))
# remote files stored on another person's computer:
# url = attachment.url
# Digest::MD5.base64digest(Net::HTTP.get(URI(url)))
end
end

View file

@ -0,0 +1,5 @@
class AddShrineImageDataToScreenshot < ActiveRecord::Migration[6.0]
def change
add_column :screenshots, :simage_data, :text
end
end

View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_10_01_000000) do
ActiveRecord::Schema.define(version: 2020_10_31_112024) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -99,6 +99,7 @@ ActiveRecord::Schema.define(version: 2020_10_01_000000) do
t.datetime "image_updated_at"
t.string "image_fingerprint"
t.integer "user_id", default: 0
t.text "image_data"
t.index ["id", "approved"], name: "id_approved"
t.index ["id", "uploaderhash"], name: "id_uploaderhash"
end

View file

@ -0,0 +1,27 @@
#require 'paperclip_shrine_synchronization'
namespace :debshots do
desc "Convert images from Paperclip (deprecated) to Shrine"
task paperclip_to_shrine: :environment do
Screenshot.find_each do |screenshot|
puts "Screenshot… #{screenshot.id}"
path = screenshot.image.path
puts "Image path… #{path}"
# Skip run if there is a Shrine attachment already (idempotency)
next if screenshot.simage
# Upload file as Shrine
screenshot.simage = File.open(path)
puts "Creating derivatives…"
screenshot.simage_derivatives!
screenshot.save!
puts "------------"
end
end
end