From 46098f2970528ca711827a3ccfeaabbc2c14535a Mon Sep 17 00:00:00 2001 From: Sojan Jose Date: Mon, 26 Oct 2020 00:31:19 +0530 Subject: [PATCH] chore: Monkey patch Azure Storage Service (#1366) Monkey patch azure storage service until the changes come into rails stable versions --- .rubocop.yml | 1 + .../azure_storage_service_patch.rb | 170 ++++++++++++++++++ lib/azure/storage.rb | 2 + .../core/auth/shared_access_signature.rb | 2 + 4 files changed, 175 insertions(+) create mode 100644 config/initializers/azure_storage_service_patch.rb create mode 100644 lib/azure/storage.rb create mode 100644 lib/azure/storage/core/auth/shared_access_signature.rb diff --git a/.rubocop.yml b/.rubocop.yml index c27b9651b..9cd50a44f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -124,3 +124,4 @@ AllCops: - 'tmp/**/*' - 'storage/**/*' - 'db/migrate/20200225162150_init_schema.rb' + - 'config/initializers/azure_storage_service_patch.rb' diff --git a/config/initializers/azure_storage_service_patch.rb b/config/initializers/azure_storage_service_patch.rb new file mode 100644 index 000000000..82a0cbd14 --- /dev/null +++ b/config/initializers/azure_storage_service_patch.rb @@ -0,0 +1,170 @@ +# TODO: Remove this once the changes comes into rails version +# https://github.com/Azure/azure-storage-ruby/issues/166#issuecomment-637696565 +# Remove this once the changes comes into rails version + +# code from current master : https://github.com/rails/rails/blob/8520cc77133d9ff642e2c393b4ee5eae2a2a28b6/activestorage/lib/active_storage/service/azure_storage_service.rb + +require 'azure/storage/blob' +require 'active_storage/service/azure_storage_service' +module ActiveStorage + # Wraps the Microsoft Azure Storage Blob Service as an Active Storage service. + # See ActiveStorage::Service for the generic API documentation that applies to all services. + class Service::AzureStorageService < Service + attr_reader :client, :container, :signer + + def initialize(storage_account_name:, storage_access_key:, container:, public: false, **options) + @client = Azure::Storage::Blob::BlobService.create(storage_account_name: storage_account_name, storage_access_key: storage_access_key, **options) + @signer = Azure::Storage::Common::Core::Auth::SharedAccessSignature.new(storage_account_name, storage_access_key) + @container = container + @public = public + end + + def upload(key, io, checksum: nil, filename: nil, content_type: nil, disposition: nil, **) + instrument :upload, key: key, checksum: checksum do + handle_errors do + content_disposition = content_disposition_with(filename: filename, type: disposition) if disposition && filename + + client.create_block_blob(container, key, IO.try_convert(io) || io, content_md5: checksum, content_type: content_type, content_disposition: content_disposition) + end + end + end + + def download(key, &block) + if block_given? + instrument :streaming_download, key: key do + stream(key, &block) + end + else + instrument :download, key: key do + handle_errors do + _, io = client.get_blob(container, key) + io.force_encoding(Encoding::BINARY) + end + end + end + end + + def download_chunk(key, range) + instrument :download_chunk, key: key, range: range do + handle_errors do + _, io = client.get_blob(container, key, start_range: range.begin, end_range: range.exclude_end? ? range.end - 1 : range.end) + io.force_encoding(Encoding::BINARY) + end + end + end + + def delete(key) + instrument :delete, key: key do + client.delete_blob(container, key) + rescue Azure::Core::Http::HTTPError => e + raise unless e.type == "BlobNotFound" + # Ignore files already deleted + end + end + + def delete_prefixed(prefix) + instrument :delete_prefixed, prefix: prefix do + marker = nil + + loop do + results = client.list_blobs(container, prefix: prefix, marker: marker) + + results.each do |blob| + client.delete_blob(container, blob.name) + end + + break unless marker = results.continuation_token.presence + end + end + end + + def exist?(key) + instrument :exist, key: key do |payload| + answer = blob_for(key).present? + payload[:exist] = answer + answer + end + end + + def url_for_direct_upload(key, expires_in:, content_type:, content_length:, checksum:) + instrument :url, key: key do |payload| + generated_url = signer.signed_uri( + uri_for(key), false, + service: "b", + permissions: "rw", + expiry: format_expiry(expires_in) + ).to_s + + payload[:url] = generated_url + + generated_url + end + end + + def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, **) + content_disposition = content_disposition_with(type: disposition, filename: filename) if filename + + { "Content-Type" => content_type, "Content-MD5" => checksum, "x-ms-blob-content-disposition" => content_disposition, "x-ms-blob-type" => "BlockBlob" } + end + + private + def private_url(key, expires_in:, filename:, disposition:, content_type:, **) + signer.signed_uri( + uri_for(key), false, + service: "b", + permissions: "r", + expiry: format_expiry(expires_in), + content_disposition: content_disposition_with(type: disposition, filename: filename), + content_type: content_type + ).to_s + end + + def public_url(key, **) + uri_for(key).to_s + end + + + def uri_for(key) + client.generate_uri("#{container}/#{key}") + end + + def blob_for(key) + client.get_blob_properties(container, key) + rescue Azure::Core::Http::HTTPError + false + end + + def format_expiry(expires_in) + expires_in ? Time.now.utc.advance(seconds: expires_in).iso8601 : nil + end + + # Reads the object for the given key in chunks, yielding each to the block. + def stream(key) + blob = blob_for(key) + + chunk_size = 5.megabytes + offset = 0 + + raise ActiveStorage::FileNotFoundError unless blob.present? + + while offset < blob.properties[:content_length] + _, chunk = client.get_blob(container, key, start_range: offset, end_range: offset + chunk_size - 1) + yield chunk.force_encoding(Encoding::BINARY) + offset += chunk_size + end + end + + def handle_errors + yield + rescue Azure::Core::Http::HTTPError => e + case e.type + when "BlobNotFound" + raise ActiveStorage::FileNotFoundError + when "Md5Mismatch" + raise ActiveStorage::IntegrityError + else + raise + end + end + end +end \ No newline at end of file diff --git a/lib/azure/storage.rb b/lib/azure/storage.rb new file mode 100644 index 000000000..f0bb9129a --- /dev/null +++ b/lib/azure/storage.rb @@ -0,0 +1,2 @@ +# https://github.com/Azure/azure-storage-ruby/issues/166#issuecomment-637696565 +# Remove this once the changes comes into rails diff --git a/lib/azure/storage/core/auth/shared_access_signature.rb b/lib/azure/storage/core/auth/shared_access_signature.rb new file mode 100644 index 000000000..ba10b61c0 --- /dev/null +++ b/lib/azure/storage/core/auth/shared_access_signature.rb @@ -0,0 +1,2 @@ +# TODO: Remove this once the changes comes into rails version +# https://github.com/Azure/azure-storage-ruby/issues/166#issuecomment-637696565