Migration to Shrine attachment handling
This commit is contained in:
parent
2d53397dda
commit
c4f5741b0c
13 changed files with 175 additions and 181 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -18,6 +18,8 @@
|
|||
/public/assets/
|
||||
/public/system/
|
||||
/public/screenshots/
|
||||
/public/screenshot/
|
||||
/public/cache/screenshot/
|
||||
/public/live/
|
||||
/storage
|
||||
|
||||
|
|
|
|||
10
Gemfile
10
Gemfile
|
|
@ -6,9 +6,13 @@ gem 'rails', '~> 6.0'
|
|||
gem 'bootsnap'
|
||||
#gem 'webpacker'
|
||||
|
||||
# To create variants (different sizes) of screenshot images saved in ActiveStorage
|
||||
# File handling
|
||||
gem "shrine", "~> 3.0"
|
||||
|
||||
# To create variants (different sizes) of screenshot images
|
||||
gem 'image_processing'
|
||||
gem 'mini_magick'
|
||||
gem 'fastimage'
|
||||
|
||||
# Use Puma as the app server
|
||||
gem 'puma', '~> 3.0'
|
||||
|
|
@ -133,7 +137,3 @@ gem 'omniauth-amazon'
|
|||
gem 'omniauth-google-oauth2'
|
||||
gem 'omniauth-github'
|
||||
gem 'omniauth-openid'
|
||||
|
||||
# Validations for ActiveStorage (which replaced Paperclip) are
|
||||
# not yet available in Rails 6.0. So this is a third-party gem.
|
||||
gem 'active_storage_validations'
|
||||
|
|
|
|||
12
Gemfile.lock
12
Gemfile.lock
|
|
@ -43,8 +43,6 @@ GEM
|
|||
erubi (~> 1.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
||||
active_storage_validations (0.8.9)
|
||||
rails (>= 5.2.0)
|
||||
activejob (6.0.3.4)
|
||||
activesupport (= 6.0.3.4)
|
||||
globalid (>= 0.3.6)
|
||||
|
|
@ -99,6 +97,7 @@ GEM
|
|||
climate_control (0.2.0)
|
||||
coderay (1.1.3)
|
||||
concurrent-ruby (1.1.7)
|
||||
content_disposition (1.0.0)
|
||||
cookies_eu (1.7.6)
|
||||
js_cookie_rails (~> 2.2.0)
|
||||
crass (1.0.6)
|
||||
|
|
@ -109,11 +108,14 @@ GEM
|
|||
railties (>= 4.1.0)
|
||||
responders
|
||||
warden (~> 1.2.3)
|
||||
down (5.2.0)
|
||||
addressable (~> 2.5)
|
||||
e2mmap (0.1.0)
|
||||
erubi (1.9.0)
|
||||
execjs (2.7.0)
|
||||
faraday (1.0.1)
|
||||
multipart-post (>= 1.2, < 3)
|
||||
fastimage (2.2.0)
|
||||
ffi (1.13.1)
|
||||
font-awesome-rails (4.7.0.5)
|
||||
railties (>= 3.2, < 6.1)
|
||||
|
|
@ -330,6 +332,9 @@ GEM
|
|||
childprocess (>= 0.5, < 4.0)
|
||||
rubyzip (>= 1.2.2)
|
||||
shellany (0.0.1)
|
||||
shrine (3.3.0)
|
||||
content_disposition (~> 1.0)
|
||||
down (~> 5.1)
|
||||
slim (4.1.0)
|
||||
temple (>= 0.7.6, < 0.9)
|
||||
tilt (>= 2.0.6, < 2.1)
|
||||
|
|
@ -405,7 +410,6 @@ PLATFORMS
|
|||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
active_storage_validations
|
||||
better_errors
|
||||
binding_of_caller
|
||||
bootsnap
|
||||
|
|
@ -414,6 +418,7 @@ DEPENDENCIES
|
|||
capybara
|
||||
cookies_eu
|
||||
devise
|
||||
fastimage
|
||||
font-awesome-rails
|
||||
foundation-rails (~> 6.6)
|
||||
guard-minitest
|
||||
|
|
@ -440,6 +445,7 @@ DEPENDENCIES
|
|||
sass-rails (~> 5.0)
|
||||
sdoc (~> 0.4.0)
|
||||
selenium-webdriver
|
||||
shrine (~> 3.0)
|
||||
slim-rails
|
||||
solargraph
|
||||
spring
|
||||
|
|
|
|||
|
|
@ -2,38 +2,21 @@ class Screenshot < ApplicationRecord
|
|||
belongs_to :package, inverse_of: :screenshots
|
||||
belongs_to :user, inverse_of: :screenshots
|
||||
|
||||
# Use paperclip gem to handle image files related to screenshots
|
||||
# After migration from Paperclip to ActiveStorage:
|
||||
has_one_attached :image
|
||||
|
||||
# Validate using 'active_storage_validations' gem
|
||||
validates :image, attached: true, content_type: { in: 'image/png', message: 'is not a valid PNG image file' }
|
||||
|
||||
# Before migration from Paperclip to ActiveStorage:
|
||||
# has_attached_file :image,
|
||||
# styles: { :large => '800x600>', :thumb => '160x120>' },
|
||||
# default_url: '/images/dummy/no-screenshots-available.svg',
|
||||
# path: ':rails_root/public/screenshots/:id_partition/:style.png',
|
||||
# url: '/screenshots/:id_partition/:style.png'
|
||||
# Paperclip
|
||||
has_attached_file :image,
|
||||
styles: { :large => '800x600>', :thumb => '160x120>' },
|
||||
default_url: '/images/dummy/no-screenshots-available.svg',
|
||||
path: ':rails_root/public/screenshots/:id_partition/:style.png',
|
||||
url: '/screenshots/:id_partition/:style.png'
|
||||
# validates_attachment_content_type :image, :content_type => 'image/png'
|
||||
# validates_with AttachmentSizeValidator, :attributes => :image, :less_than => 5.megabytes
|
||||
# # validate :validate_image_is_unique
|
||||
# validate :validate_image_is_unique
|
||||
# validates :delete_reason, length: { in: 5..100 }, allow_nil: true
|
||||
|
||||
# Validator that checks if the image has already been uploaded.
|
||||
# A generic uniqueness validator does not work because it would attribute
|
||||
# the error to the :image_fingerprint field and not the actual :image field.
|
||||
# Currently this check makes sure that the same screenshot is not uploaded
|
||||
# twice for the same package. However it allows the screenshot to be uploaded
|
||||
# for two different packages. Restricting that further may happen later.
|
||||
def validate_image_is_unique
|
||||
# Look for images with the same checksum / image_fingerprint
|
||||
if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
|
||||
unless screenshot.id == self.id
|
||||
errors.add(:image, "has already been uploaded for this package")
|
||||
end
|
||||
end
|
||||
end
|
||||
# Shrine
|
||||
# include PaperclipShrineSynchronization # needs to be after `has_attached_file`
|
||||
include ImageUploader::Attachment(:simage) # adds an `simage` virtual attribute
|
||||
|
||||
|
||||
# Calculate how many days ago this screenshot has been uploaded
|
||||
def age_days
|
||||
|
|
@ -180,4 +163,32 @@ class Screenshot < ApplicationRecord
|
|||
).processed
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# Validator that checks if the image has already been uploaded.
|
||||
# A generic uniqueness validator does not work because it would attribute
|
||||
# the error to the :image_fingerprint field and not the actual :image field.
|
||||
# Currently this check makes sure that the same screenshot is not uploaded
|
||||
# twice for the same package. However it allows the screenshot to be uploaded
|
||||
# for two different packages. Restricting that further may happen later.
|
||||
def validate_image_is_unique
|
||||
# Look for images with the same checksum / image_fingerprint
|
||||
if screenshot=Screenshot.find_by(image_fingerprint: image_fingerprint, package_id: self.package.id)
|
||||
unless screenshot.id == self.id
|
||||
errors.add(:image, "has already been uploaded for this package")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Check that the uploaded file can be handled by ImageMagick and is not broken
|
||||
# def image_intact?
|
||||
# begin
|
||||
# #MiniMagick::Image.open(self.image.blob.open)
|
||||
# MiniMagick::Image.read(self.image.download)
|
||||
# rescue MiniMagick::Invalid => exc
|
||||
# error.add(:image, 'appears to be a broken image file')
|
||||
# end
|
||||
# end
|
||||
|
||||
end
|
||||
|
|
|
|||
49
app/uploaders/image_uploader.rb
Normal file
49
app/uploaders/image_uploader.rb
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
# This is a subclass of Shrine base that will be further configured for it's requirements.
|
||||
# This will be included in the model to manage the file.
|
||||
|
||||
class ImageUploader < Shrine
|
||||
ALLOWED_TYPES = %w[image/jpeg image/png image/webp]
|
||||
MAX_SIZE = 5*1024*1024 # 5 MB
|
||||
MAX_DIMENSIONS = [3000, 3000] # 3000x3000
|
||||
|
||||
plugin :remove_attachment
|
||||
plugin :pretty_location
|
||||
plugin :validation_helpers
|
||||
plugin :store_dimensions, log_subscriber: nil
|
||||
# plugin :derivation_endpoint, prefix: "derivations/image"
|
||||
|
||||
# File validations (requires `validation_helpers` plugin)
|
||||
Attacher.validate do
|
||||
validate_size 0..MAX_SIZE
|
||||
|
||||
if validate_mime_type ALLOWED_TYPES
|
||||
validate_max_dimensions MAX_DIMENSIONS
|
||||
end
|
||||
end
|
||||
|
||||
# Thumbnails processor (requires `derivatives` plugin)
|
||||
Attacher.derivatives do |original|
|
||||
puts "original=#{original}"
|
||||
magick = ImageProcessing::MiniMagick.source(original)
|
||||
{
|
||||
small: magick.resize_to_limit!(160, 120),
|
||||
medium: magick.resize_to_limit!(800, 600),
|
||||
large: magick.composite!('public/logo/watermark.png', gravity: 'east')
|
||||
}
|
||||
# GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
|
||||
|
||||
# THUMBNAILS.transform_values do |(width, height)|
|
||||
# GenerateThumbnail.call(original, width, height) # lib/generate_thumbnail.rb
|
||||
# end
|
||||
end
|
||||
|
||||
# Default to dynamic thumbnail URL (requires `default_url` plugin)
|
||||
Attacher.default_url do |derivative: nil, **|
|
||||
file&.derivation_url(:thumbnail, *THUMBNAILS.fetch(derivative)) if derivative
|
||||
end
|
||||
|
||||
# Dynamic thumbnail definition (requires `derivation_endpoint` plugin)
|
||||
# derivation :thumbnail do |file, width, height|
|
||||
# GenerateThumbnail.call(file, width.to_i, height.to_i) # lib/generate_thumbnail.rb
|
||||
# end
|
||||
end
|
||||
|
|
@ -4,7 +4,8 @@ a.black href=package_path(name: pkg.name)
|
|||
- if pkg.screenshots_approved.any?
|
||||
// TODO: smarter selection of the most useful screenshot instead of taking the first one
|
||||
- screenshot = pkg.screenshots.first
|
||||
= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
|
||||
/= image_tag(screenshot.image.variant(resize_to_limit: [160,120]), alt: screenshot.caption, class: 'thumbnail')
|
||||
= image_tag(screenshot.simage_url(:small), alt: screenshot.caption, class: 'thumbnail')
|
||||
- else
|
||||
img.screenshot.thumbnail src="/images/dummy/no-screenshots-upload-one.svg"
|
||||
div
|
||||
|
|
|
|||
14
app/views/packages/_medium_image.slim
Normal file
14
app/views/packages/_medium_image.slim
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/ Show image in medium size with zoom option and caption below
|
||||
a href =url_for(screenshot.simage_url(:large)) rel='fancybox-thumb' title=screenshot.caption data-fancybox='gallery' data-caption=screenshot_caption(screenshot)
|
||||
.image-with-zoom-icon
|
||||
= image_tag(url_for(screenshot.simage_url(:medium)), alt: screenshot.caption, class: 'thumbnail')
|
||||
|
||||
.magnifying-glass-icon
|
||||
i.fa.fa-search
|
||||
|
||||
.imgcaption
|
||||
= screenshot.description
|
||||
|
||||
/ Display management buttons for admins only
|
||||
- if user_signed_in? and current_user.is_admin?
|
||||
= render(partial: 'admin_dropdown', locals: {screenshot: screenshot})
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# Be sure to restart your server when you modify this file.
|
||||
#
|
||||
# This file contains migration options to ease your Rails 5.2 upgrade.
|
||||
#
|
||||
# Once upgraded flip defaults one by one to migrate to the new default.
|
||||
#
|
||||
# Read the Guide for Upgrading Ruby on Rails for more info on each option.
|
||||
|
||||
# Make Active Record use stable #cache_key alongside new #cache_version method.
|
||||
# This is needed for recyclable cache keys.
|
||||
# Rails.application.config.active_record.cache_versioning = true
|
||||
|
||||
# Use AES-256-GCM authenticated encryption for encrypted cookies.
|
||||
# Also, embed cookie expiry in signed or encrypted cookies for increased security.
|
||||
#
|
||||
# This option is not backwards compatible with earlier Rails versions.
|
||||
# It's best enabled when your entire app is migrated and stable on 5.2.
|
||||
#
|
||||
# Existing cookies will be converted on read then written with the new scheme.
|
||||
# Rails.application.config.action_dispatch.use_authenticated_cookie_encryption = true
|
||||
|
||||
# Use AES-256-GCM authenticated encryption as default cipher for encrypting messages
|
||||
# instead of AES-256-CBC, when use_authenticated_message_encryption is set to true.
|
||||
# Rails.application.config.active_support.use_authenticated_message_encryption = true
|
||||
|
||||
# Add default protection from forgery to ActionController::Base instead of in
|
||||
# ApplicationController.
|
||||
# Rails.application.config.action_controller.default_protect_from_forgery = true
|
||||
|
||||
# Store boolean values are in sqlite3 databases as 1 and 0 instead of 't' and
|
||||
# 'f' after migrating old data.
|
||||
# Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
|
||||
|
||||
# Use SHA-1 instead of MD5 to generate non-sensitive digests, such as the ETag header.
|
||||
# Rails.application.config.active_support.use_sha1_digests = true
|
||||
|
||||
# Make `form_with` generate id attributes for any generated HTML tags.
|
||||
# Rails.application.config.action_view.form_with_generates_ids = true
|
||||
21
config/initializers/shrine.rb
Normal file
21
config/initializers/shrine.rb
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# See also: https://github.com/erikdahlstrand/shrine-rails-example/blob/master/config/initializers/shrine.rb
|
||||
|
||||
require "shrine"
|
||||
require "shrine/storage/file_system"
|
||||
|
||||
# both `cache` and `store` storages are needed
|
||||
Shrine.storages = {
|
||||
# Saves to ./public/screenshot/ID/image/…
|
||||
cache: Shrine::Storage::FileSystem.new("public", prefix: "cache"),
|
||||
store: Shrine::Storage::FileSystem.new("public", prefix: "/"),
|
||||
}
|
||||
|
||||
# See plugin documentation at https://shrinerb.com/docs/plugins/activerecord
|
||||
Shrine.plugin :activerecord
|
||||
Shrine.plugin :instrumentation
|
||||
Shrine.plugin :determine_mime_type, analyzer: :marcel, log_subscriber: nil
|
||||
Shrine.plugin :cached_attachment_data
|
||||
Shrine.plugin :restore_cached_data
|
||||
Shrine.plugin :derivatives # up front processing
|
||||
Shrine.plugin :derivation_endpoint, # on-the-fly processing
|
||||
secret_key: Rails.application.secret_key_base
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
# Convert the screenshots from the deprecated Paperclip gem schema
|
||||
# to the new ActiveStorage schema built into Rails 5.2.
|
||||
|
||||
Dir[Rails.root.join("app/models/**/*.rb")].sort.each { |file| require file }
|
||||
|
||||
class ConvertToActiveStorage < ActiveRecord::Migration[5.2]
|
||||
require 'open-uri'
|
||||
|
||||
def up
|
||||
puts "Running up..."
|
||||
# postgres
|
||||
get_blob_id = 'LASTVAL()'
|
||||
# mysql / mariadb
|
||||
# get_blob_id = 'LAST_INSERT_ID()'
|
||||
# sqlite
|
||||
# get_blob_id = 'LAST_INSERT_ROWID()'
|
||||
|
||||
puts "Preparing queries"
|
||||
|
||||
active_storage_blob_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL)
|
||||
INSERT INTO active_storage_blobs (
|
||||
key, filename, content_type, metadata, byte_size, checksum, created_at
|
||||
) VALUES ($1, $2, $3, '{}', $4, $5, $6)
|
||||
SQL
|
||||
|
||||
active_storage_attachment_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL)
|
||||
INSERT INTO active_storage_attachments (
|
||||
name, record_type, record_id, blob_id, created_at
|
||||
) VALUES ($1, $2, $3, #{get_blob_id}, $4)
|
||||
SQL
|
||||
|
||||
|
||||
Rails.application.eager_load!
|
||||
models = ActiveRecord::Base.descendants.reject(&:abstract_class?)
|
||||
|
||||
transaction do
|
||||
models.each do |model|
|
||||
attachments = model.column_names.map do |c|
|
||||
if c =~ /(.+)_file_name$/
|
||||
$1
|
||||
end
|
||||
end.compact
|
||||
|
||||
if attachments.blank?
|
||||
next
|
||||
end
|
||||
|
||||
model.find_each.each do |instance|
|
||||
puts "Model: #{instance}"
|
||||
attachments.each do |attachment|
|
||||
puts "Attachment: #{attachment}"
|
||||
if instance.send(attachment).path.blank?
|
||||
next
|
||||
end
|
||||
|
||||
ActiveRecord::Base.connection.raw_connection.exec_prepared(
|
||||
'active_storage_blob_statement', [
|
||||
key(instance, attachment),
|
||||
instance.send("#{attachment}_file_name"),
|
||||
instance.send("#{attachment}_content_type"),
|
||||
instance.send("#{attachment}_file_size"),
|
||||
checksum(instance.send(attachment)),
|
||||
instance.updated_at.iso8601
|
||||
])
|
||||
|
||||
ActiveRecord::Base.connection.raw_connection.exec_prepared(
|
||||
'active_storage_attachment_statement', [
|
||||
attachment,
|
||||
model.name,
|
||||
instance.id,
|
||||
instance.updated_at.iso8601,
|
||||
])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def down
|
||||
raise ActiveRecord::IrreversibleMigration
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def key(instance, attachment)
|
||||
SecureRandom.uuid
|
||||
# Alternatively:
|
||||
# filename = instance.send("#{attachment}_file_name")
|
||||
# klass = instance.class.table_name
|
||||
# id = instance.id
|
||||
# id_partition = ("%09d".freeze % id).scan(/\d{3}/).join("/".freeze)
|
||||
|
||||
# "#{klass}/#{attachment.pluralize}/#{id_partition}/original/#{filename}"
|
||||
end
|
||||
|
||||
def checksum(attachment)
|
||||
# local files stored on disk:
|
||||
url = attachment.path
|
||||
Digest::MD5.base64digest(File.read(url))
|
||||
|
||||
# remote files stored on another person's computer:
|
||||
# url = attachment.url
|
||||
# Digest::MD5.base64digest(Net::HTTP.get(URI(url)))
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
class AddShrineImageDataToScreenshot < ActiveRecord::Migration[6.0]
|
||||
def change
|
||||
add_column :screenshots, :simage_data, :text
|
||||
end
|
||||
end
|
||||
|
|
@ -10,7 +10,7 @@
|
|||
#
|
||||
# It's strongly recommended that you check this file into your version control system.
|
||||
|
||||
ActiveRecord::Schema.define(version: 2020_10_01_000000) do
|
||||
ActiveRecord::Schema.define(version: 2020_10_31_112024) do
|
||||
|
||||
# These are extensions that must be enabled in order to support this database
|
||||
enable_extension "plpgsql"
|
||||
|
|
@ -99,6 +99,7 @@ ActiveRecord::Schema.define(version: 2020_10_01_000000) do
|
|||
t.datetime "image_updated_at"
|
||||
t.string "image_fingerprint"
|
||||
t.integer "user_id", default: 0
|
||||
t.text "image_data"
|
||||
t.index ["id", "approved"], name: "id_approved"
|
||||
t.index ["id", "uploaderhash"], name: "id_uploaderhash"
|
||||
end
|
||||
|
|
|
|||
27
lib/tasks/paperclip_to_shrine.rake
Normal file
27
lib/tasks/paperclip_to_shrine.rake
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
#require 'paperclip_shrine_synchronization'
|
||||
|
||||
namespace :debshots do
|
||||
desc "Convert images from Paperclip (deprecated) to Shrine"
|
||||
|
||||
task paperclip_to_shrine: :environment do
|
||||
|
||||
Screenshot.find_each do |screenshot|
|
||||
puts "Screenshot… #{screenshot.id}"
|
||||
path = screenshot.image.path
|
||||
puts "Image path… #{path}"
|
||||
|
||||
# Skip run if there is a Shrine attachment already (idempotency)
|
||||
next if screenshot.simage
|
||||
|
||||
# Upload file as Shrine
|
||||
screenshot.simage = File.open(path)
|
||||
|
||||
puts "Creating derivatives…"
|
||||
screenshot.simage_derivatives!
|
||||
|
||||
screenshot.save!
|
||||
puts "------------"
|
||||
end
|
||||
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Add a link
Reference in a new issue