Verified Commit 806dbac0 authored by Jiri Strojil's avatar Jiri Strojil 💬 Committed by Kaspar Vollenweider
Browse files

Feat/activestorage migration

parent 053f3638
......@@ -10,7 +10,7 @@
= render 'contacts/show', contact: @profile.contact
tr
td= t_attr(:avatar)
td= image_tag @profile.avatar.url(:thumb) if @profile.avatar.present?
td= image_tag rails_representation_url(@profile.avatar_thumb) if @profile.avatar.present?
tr
td= t_attr(:profession)
td= @profile.profession
......
- if @current_assignments.any? || @current_group_assignments.any?
- if @current_assignments&.any? || @current_group_assignments&.any?
h2.small Aktuelle Einsätze
- if @current_assignments.any?
- if @current_assignments&.any?
h3.small#assignments Begleitungen
= render 'assignments/client_volunteer_index', assignments: @current_assignments
......@@ -8,13 +8,13 @@
h3.small#assignments Gruppenangebote
= render 'group_assignments/volunteer_group_assignments', group_assignments: @current_group_assignments, editable: true
- if @archived_assignments.any? || @archived_group_assignments.any?
- if @archived_assignments&.any? || @archived_group_assignments&.any?
h2.small Archivierte Einsätze
- if @archived_assignments.any?
- if @archived_assignments&.any?
h3.small Begleitungen
= render 'assignment_logs/volunteer_index', assignment_logs: @archived_assignments
- if @archived_group_assignments.any?
- if @archived_group_assignments&.any?
h3.small Gruppenangebote
= render 'group_assignment_logs/volunteer_index', group_assignment_logs: @archived_group_assignments
hr
......@@ -38,7 +38,7 @@ h2.small Persönlicher Hintergrund
td= l(@volunteer.birth_year) if @volunteer.birth_year.present?
tr
td= t_attr(:avatar)
td= image_tag @volunteer.avatar.url(:thumb) if @volunteer.avatar.present?
td= image_tag rails_representation_url(@volunteer.avatar_thumb) if @volunteer.avatar.present?
tr
td= t_attr(:nationality)
td= nationality_name(@volunteer.nationality)
......
......@@ -8,6 +8,7 @@ require 'rails/all'
Bundler.require(*Rails.groups)
require 'webdrivers/chromedriver'
require 'active_storage/engine'
Webdrivers.install_dir = '.ci-cache/webdrivers'
module Aoz
......
......@@ -42,7 +42,11 @@ Rails.application.configure do
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
# Store uploaded files on the local file system (see config/storage.yml for options).
config.active_storage.service = :local
config.active_storage.service = if ENV.fetch('GOOGLE_PROJECT_ID', '').present?
:google
else
:local
end
# Mount Action Cable outside main process or domain.
# config.action_cable.mount_path = nil
......
......@@ -6,29 +6,18 @@ local:
service: Disk
root: <%= Rails.root.join("storage") %>
# Use rails credentials:edit to set the AWS secrets (as aws:access_key_id|secret_access_key)
# amazon:
# service: S3
# access_key_id: <%= Rails.application.credentials.dig(:aws, :access_key_id) %>
# secret_access_key: <%= Rails.application.credentials.dig(:aws, :secret_access_key) %>
# region: us-east-1
# bucket: your_own_bucket
# Remember not to checkin your GCS keyfile to a repository
# google:
# service: GCS
# project: your_project
# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %>
# bucket: your_own_bucket
# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key)
# microsoft:
# service: AzureStorage
# storage_account_name: your_account_name
# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %>
# container: your_container_name
# mirror:
# service: Mirror
# primary: local
# mirrors: [ amazon, google, microsoft ]
google:
service: GCS
credentials:
type: "service_account"
project_id: "<%= ENV.fetch('GOOGLE_PROJECT_ID', '') %>"
private_key_id: "<%= ENV.fetch('GOOGLE_PRIVATE_KEY_ID', '') %>"
private_key: "<%= ENV.fetch('GOOGLE_PRIVATE_KEY', '').gsub("\n", '\n') %>"
client_email: "<%= ENV.fetch('GOOGLE_CLIENT_EMAIL', '') %>"
client_id: "<%= ENV.fetch('GOOGLE_CLIENT_ID', '') %>"
auth_uri: "https://accounts.google.com/o/oauth2/auth"
token_uri: "https://accounts.google.com/o/oauth2/token"
auth_provider_x509_cert_url: "https://www.googleapis.com/oauth2/v1/certs"
client_x509_cert_url: "<%= ENV.fetch('GOOGLE_CLIENT_X509_CERT_URL', '') %>"
project: "<%= ENV.fetch('GOOGLE_PROJECT_ID', '') %>"
bucket: "<%= ENV.fetch('GOOGLE_BUCKET', '') %>"
\ No newline at end of file
# This migration comes from active_storage (originally 20170806125915)
class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
def change
create_table :active_storage_blobs do |t|
t.string :key, null: false
t.string :filename, null: false
t.string :content_type
t.text :metadata
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.index [ :key ], unique: true
end
create_table :active_storage_attachments do |t|
t.string :name, null: false
t.references :record, null: false, polymorphic: true, index: false
t.references :blob, null: false
t.datetime :created_at, null: false
t.index [ :record_type, :record_id, :name, :blob_id ], name: "index_active_storage_attachments_uniqueness", unique: true
t.foreign_key :active_storage_blobs, column: :blob_id
end
end
end
class ConvertToActiveStorage < ActiveRecord::Migration[5.2]
require 'open-uri'
def up
# postgres
get_blob_id = 'LASTVAL()'
# mariadb
# get_blob_id = 'LAST_INSERT_ID()'
# sqlite
# get_blob_id = 'LAST_INSERT_ROWID()'
active_storage_blob_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_blob_statement', <<-SQL)
INSERT INTO active_storage_blobs (
key, filename, content_type, metadata, byte_size, checksum, created_at
) VALUES ($1, $2, $3, '{}', $4, $5, $6)
SQL
active_storage_attachment_statement = ActiveRecord::Base.connection.raw_connection.prepare('active_storage_attachment_statement', <<-SQL)
INSERT INTO active_storage_attachments (
name, record_type, record_id, blob_id, created_at
) VALUES ($1, $2, $3, #{"(SELECT max(id) from active_storage_blobs)"}, $4)
SQL
Rails.application.eager_load!
models = ActiveRecord::Base.descendants.reject(&:abstract_class?)
disabled_models = ["Feedback", "ActionText::RichText", "ActionMailbox::InboundEmail", "ActiveStorage::Blob", "ActiveStorage::Attachment"]
transaction do
models.reject { |model| disabled_models.include? model.name }.each do |model|
attachments = model.column_names.map do |c|
if c =~ /(.+)_file_name$/
$1
end
end.compact
if attachments.blank?
next
end
model.find_each.each do |instance|
attachments.each do |attachment|
filename = instance.send("#{attachment}_file_name")
full_path = "#{Rails.root}/public/system/#{ActiveSupport::Inflector.pluralize(attachment)}/#{instance.id}/original/#{filename.to_s}"
if filename.blank? || !File.file?(full_path)
next
end
ActiveRecord::Base.connection.raw_connection.exec_prepared(
'active_storage_blob_statement', [
key(instance, attachment),
filename,
instance.send("#{attachment}_content_type"),
instance.send("#{attachment}_file_size"),
checksum(full_path),
instance.send("#{attachment}_updated_at").iso8601
])
ActiveRecord::Base.connection.raw_connection.exec_prepared(
'active_storage_attachment_statement', [
attachment,
model.name,
instance.id,
instance.send("#{attachment}_updated_at").iso8601,
])
end
end
end
end
end
def down
raise ActiveRecord::IrreversibleMigration
end
private
def key(instance, attachment)
SecureRandom.uuid
# Alternatively:
# instance.send("#{attachment}_file_name")
end
def checksum(filename)
# local files stored on disk:
Digest::MD5.base64digest(File.read(filename))
# remote files stored on another person's computer:
# url = attachment.url
# Digest::MD5.base64digest(Net::HTTP.get(URI(url)))
end
end
\ No newline at end of file
......@@ -10,11 +10,32 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 2020_05_26_092317) do
ActiveRecord::Schema.define(version: 2020_05_28_133812) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
create_table "active_storage_attachments", force: :cascade do |t|
t.string "name", null: false
t.string "record_type", null: false
t.bigint "record_id", null: false
t.bigint "blob_id", null: false
t.datetime "created_at", null: false
t.index ["blob_id"], name: "index_active_storage_attachments_on_blob_id"
t.index ["record_type", "record_id", "name", "blob_id"], name: "index_active_storage_attachments_uniqueness", unique: true
end
create_table "active_storage_blobs", force: :cascade do |t|
t.string "key", null: false
t.string "filename", null: false
t.string "content_type"
t.text "metadata"
t.bigint "byte_size", null: false
t.string "checksum", null: false
t.datetime "created_at", null: false
t.index ["key"], name: "index_active_storage_blobs_on_key", unique: true
end
create_table "assignment_logs", force: :cascade do |t|
t.bigint "assignment_id"
t.bigint "volunteer_id"
......@@ -866,6 +887,7 @@ ActiveRecord::Schema.define(version: 2020_05_26_092317) do
t.index ["user_id"], name: "index_volunteers_on_user_id"
end
add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
add_foreign_key "assignment_logs", "assignments"
add_foreign_key "assignment_logs", "clients"
add_foreign_key "assignment_logs", "volunteers"
......
namespace :active_storage do
desc 'ActiveStorage actions'
task move_paperclip_files: :environment do
ActiveStorage::Attachment.find_each do |attachment|
name = attachment.name
filename = attachment.blob.filename
source = "#{Rails.root}/public/system/#{ActiveSupport::Inflector.pluralize(name)}/#{attachment.record_id}/original/#{filename}"
dest_dir = File.join(
'storage',
attachment.blob.key.first(2),
attachment.blob.key.first(4).last(2)
)
dest = File.join(dest_dir, attachment.blob.key)
# file no longer exists
next unless File.exist? source
FileUtils.mkdir_p(dest_dir)
puts "Moving #{source} to #{dest}"
FileUtils.cp(source, dest)
end
end
def migrate(from, to)
config_file = Pathname.new(Rails.root.join('config/storage.yml'))
configs = YAML.safe_load(ERB.new(config_file.read).result) || {}
from_service = ActiveStorage::Service.configure from, configs
to_service = ActiveStorage::Service.configure to, configs
ActiveStorage::Blob.service = from_service
puts "#{ActiveStorage::Blob.count} Blobs to go..."
ActiveStorage::Blob.find_each do |blob|
print '.'
file = Tempfile.new("file#{Time.now.to_f}")
file.binmode
file << blob.download
file.rewind
checksum = blob.checksum
to_service.upload(blob.key, file, checksum: checksum)
file.close
file.unlink
rescue Errno::ENOENT
puts "Rescued by Errno::ENOENT statement. ID: #{blob.id} / Key: #{blob.key}"
next
rescue ActiveStorage::FileNotFoundError
puts "Rescued by FileNotFoundError. ID: #{blob.id} / Key: #{blob.key}"
next
end
end
task migrate_to_s3: :environment do
migrate(:local, :google)
end
end
# rubocop:disable Style/MixinUsage
include ActionDispatch::TestProcess
# rubocop:enable Style/MixinUsage
FactoryBot.define do
factory :document do
title { FFaker::Name }
......@@ -5,6 +9,10 @@ FactoryBot.define do
category2 { FFaker::Name }
category3 { FFaker::Name }
category4 { FFaker::Name }
file { File.new(File.join(Rails.root, 'test/fixtures/sample.pdf')) }
file do
fixture_file_upload(
File.join(Rails.root, 'test/fixtures/sample.pdf'), 'application/pdf'
)
end
end
end
......@@ -13,6 +13,10 @@ class ActionMailer::TestCase
include GroupOfferAndAssignment
end
FactoryBot::SyntaxRunner.class_eval do
include ActionDispatch::TestProcess
end
class ActiveSupport::TestCase
include FactoryBot::Syntax::Methods
include Warden::Test::Helpers
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment