batchy yay

This commit is contained in:
End Nightshade 2026-02-10 11:14:41 -07:00
parent def7b0511f
commit 53f80ef244
No known key found for this signature in database
6 changed files with 250 additions and 176 deletions

View file

@ -87,7 +87,7 @@ class Components::Uploads::Index < Components::Base
if query.present? if query.present?
"Try a different search query" "Try a different search query"
else else
"Drag and drop files anywhere on this page, or use the Upload button" "Drag and drop files anywhere on this page, or use the Upload button (up to 40 files at once)"
end end
end end
end end
@ -101,7 +101,7 @@ class Components::Uploads::Index < Components::Base
def dropzone_form def dropzone_form
form_with url: uploads_path, method: :post, multipart: true, data: { dropzone_form: true } do form_with url: uploads_path, method: :post, multipart: true, data: { dropzone_form: true } do
input(type: "file", name: "file", id: "dropzone-file-input", data: { dropzone_input: true }, style: "display: none;") input(type: "file", name: "files[]", id: "dropzone-file-input", multiple: true, data: { dropzone_input: true }, style: "display: none;")
end end
end end
end end

View file

@ -1,82 +1,157 @@
# frozen_string_literal: true # frozen_string_literal: true
class UploadsController < ApplicationController module API
before_action :set_upload, only: [ :destroy ] module V4
class UploadsController < ApplicationController
before_action :check_quota, only: [ :create, :create_from_url ]
def index # POST /api/v4/upload
@uploads = current_user.uploads.includes(:blob).recent def create
file = params[:file]
if params[:query].present? unless file.present?
@uploads = @uploads.search_by_filename(params[:query]) render json: { error: "Missing file parameter" }, status: :bad_request
return
end
content_type = Marcel::MimeType.for(file.tempfile, name: file.original_filename) || file.content_type || "application/octet-stream"
upload_id = SecureRandom.uuid_v7
sanitized_filename = ActiveStorage::Filename.new(file.original_filename).sanitized
storage_key = "#{upload_id}/#{sanitized_filename}"
blob = ActiveStorage::Blob.create_and_upload!(
io: file.tempfile,
filename: file.original_filename,
content_type: content_type,
key: storage_key
)
upload = current_user.uploads.create!(id: upload_id, blob: blob, provenance: :api)
render json: upload_json(upload), status: :created
rescue => e
render json: { error: "Upload failed: #{e.message}" }, status: :unprocessable_entity
end
# POST /api/v4/uploads (batch)
def create_batch
files = params[:files]
unless files.present? && files.is_a?(Array)
render json: { error: "Missing files[] parameter" }, status: :bad_request
return
end
if files.size > BatchUploadService::MAX_FILES_PER_BATCH
render json: {
error: "Too many files",
detail: "Maximum #{BatchUploadService::MAX_FILES_PER_BATCH} files per batch, got #{files.size}"
}, status: :bad_request
return
end
service = BatchUploadService.new(user: current_user, provenance: :api)
result = service.process_files(files)
response = {
uploads: result.uploads.map { |u| upload_json(u) },
failed: result.failed.map { |f| { filename: f.filename, reason: f.reason } }
}
status = result.uploads.any? ? :created : :unprocessable_entity
render json: response, status: status
end
# POST /api/v4/upload_from_url
def create_from_url
url = params[:url]
unless url.present?
render json: { error: "Missing url parameter" }, status: :bad_request
return
end
download_auth = request.headers["X-Download-Authorization"]
upload = Upload.create_from_url(url, user: current_user, provenance: :api, original_url: url, authorization: download_auth)
quota_service = QuotaService.new(current_user)
unless quota_service.can_upload?(0)
if current_user.total_storage_bytes > quota_service.current_policy.max_total_storage
upload.destroy!
usage = quota_service.current_usage
render json: quota_error_json(usage), status: :payment_required
return
end
end
render json: upload_json(upload), status: :created
rescue => e
render json: { error: "Upload failed: #{e.message}" }, status: :unprocessable_entity
end
# PATCH /api/v4/uploads/:id/rename
def rename
upload = current_user.uploads.find(params[:id])
new_filename = params[:filename].to_s.strip
if new_filename.blank?
render json: { error: "Missing filename parameter" }, status: :bad_request
return
end
upload.rename!(new_filename)
render json: upload_json(upload)
rescue ActiveRecord::RecordNotFound
render json: { error: "Upload not found" }, status: :not_found
rescue => e
render json: { error: "Rename failed: #{e.message}" }, status: :unprocessable_entity
end
private
def check_quota
if params[:file].present?
file_size = params[:file].size
quota_service = QuotaService.new(current_user)
policy = quota_service.current_policy
if file_size > policy.max_file_size
usage = quota_service.current_usage
render json: quota_error_json(usage, "File size exceeds your limit of #{ActiveSupport::NumberHelper.number_to_human_size(policy.max_file_size)} per file"), status: :payment_required
return
end
unless quota_service.can_upload?(file_size)
usage = quota_service.current_usage
render json: quota_error_json(usage), status: :payment_required
nil
end
end
end
def quota_error_json(usage, custom_message = nil)
{
error: custom_message || "Storage quota exceeded",
quota: {
storage_used: usage[:storage_used],
storage_limit: usage[:storage_limit],
quota_tier: usage[:policy],
percentage_used: usage[:percentage_used]
}
}
end
def upload_json(upload)
{
id: upload.id,
filename: upload.filename.to_s,
size: upload.byte_size,
content_type: upload.content_type,
url: upload.cdn_url,
created_at: upload.created_at.iso8601
}
end
end end
@uploads = @uploads.page(params[:page]).per(50)
end
def create
uploaded_files = extract_uploaded_files
if uploaded_files.empty?
redirect_to uploads_path, alert: "Please select at least one file to upload."
return
end
if uploaded_files.size > BatchUploadService::MAX_FILES_PER_BATCH
redirect_to uploads_path, alert: "Too many files selected. Max #{BatchUploadService::MAX_FILES_PER_BATCH} files allowed per upload."
return
end
service = BatchUploadService.new(user: current_user, provenance: :web)
result = service.process_files(uploaded_files)
flash_message = build_flash_message(result)
if result.uploads.any?
redirect_to uploads_path, notice: flash_message
else
redirect_to uploads_path, alert: flash_message
end
rescue StandardError => e
event = Sentry.capture_exception(e)
redirect_to uploads_path, alert: "Upload failed: #{e.message} (Error ID: #{event&.event_id})"
end
def destroy
authorize @upload
@upload.destroy!
redirect_back fallback_location: uploads_path, notice: "Upload deleted successfully."
rescue Pundit::NotAuthorizedError
redirect_back fallback_location: uploads_path, alert: "You are not authorized to delete this upload."
end
private
def extract_uploaded_files
files = []
files.concat(Array(params[:files])) if params[:files].present?
files << params[:file] if params[:file].present?
files.reject(&:blank?)
end
def build_flash_message(result)
parts = []
if result.uploads.any?
count = result.uploads.size
filenames = result.uploads.map { |u| u.filename.to_s }.join(", ")
parts << "Uploaded #{count} #{'file'.pluralize(count)}: #{filenames}"
end
if result.failed.any?
failures = result.failed.map { |f| "#{f.filename} (#{f.reason})" }.join("; ")
parts << "Failed: #{failures}"
end
parts.join(". ")
end
def set_upload
@upload = Upload.find(params[:id])
end end
end end

View file

@ -1,7 +1,7 @@
# frozen_string_literal: true # frozen_string_literal: true
class UploadsController < ApplicationController class UploadsController < ApplicationController
before_action :set_upload, only: [ :destroy ] before_action :set_upload, only: [ :update, :destroy ]
def index def index
@uploads = current_user.uploads.includes(:blob).recent @uploads = current_user.uploads.includes(:blob).recent
@ -31,78 +31,67 @@ class UploadsController < ApplicationController
flash_message = build_flash_message(result) flash_message = build_flash_message(result)
if result.uplaods.any? if result.uploads.any?
redirect_to uploads_path, notice: flash_message redirect_to uploads_path, notice: flash_message
else else
redirect_to uploads_path, alert: flash_message redirect_to uploads_path, alert: flash_message
end end
rescue StandardError => e
event = Sentry.capture_exception(e)
redirect_to uploads_path, alert: "Upload failed: #{e.message} (Error ID: #{event&.event_id})"
end
def destroy
authorize @upload
@upload.destroy!
redirect_back fallback_location: uploads_path, notice: "Upload deleted successfully."
rescue Pundit::NotAuthorizedError
redirect_back fallback_location: uploads_path, alert: "You are not authorized to delete this upload."
end
private
def extract_uploaded_files
files=[]
# mult files via files[] param
if params[:files].present?
files.concat(Array(params[:files]))
end
if params[:file].present?
files << params[:file]
end
files.reject(&:blank?)
end
content_type = Marcel::MimeType.for(uploaded_file.tempfile, name: uploaded_file.original_filename) || uploaded_file.content_type || "application/octet-stream"
redirect_to uploads_path, notice: "File uploaded successfully!"
rescue StandardError => e rescue StandardError => e
event = Sentry.capture_exception(e) event = Sentry.capture_exception(e)
redirect_to uploads_path, alert: "Upload failed: #{e.message} (Error ID: #{event&.event_id})" redirect_to uploads_path, alert: "Upload failed: #{e.message} (Error ID: #{event&.event_id})"
end end
files.reject(&:blank?) def update
authorize @upload
new_filename = params[:filename].to_s.strip
if new_filename.blank?
redirect_to uploads_path, alert: "Filename can't be blank."
return
end
@upload.rename!(new_filename)
redirect_to uploads_path, notice: "Renamed to #{@upload.filename}"
rescue Pundit::NotAuthorizedError
redirect_to uploads_path, alert: "You are not authorized to rename this upload."
end
def destroy
authorize @upload
@upload.destroy!
redirect_back fallback_location: uploads_path, notice: "Upload deleted successfully."
rescue Pundit::NotAuthorizedError
redirect_back fallback_location: uploads_path, alert: "You are not authorized to delete this upload."
end
private
def extract_uploaded_files
files = []
files.concat(Array(params[:files])) if params[:files].present?
files << params[:file] if params[:file].present?
files.reject(&:blank?)
end end
def build_flash_message(result) def build_flash_message(result)
messages = [] parts = []
if result.uploads.any? if result.uploads.any?
messages << "#{result.uploads.size} file(s) uploaded successfully" count = result.uploads.size
filenames = result.uploads.map { |u| u.filename.to_s }.join(", ")
parts << "Uploaded #{count} #{'file'.pluralize(count)}: #{filenames}"
end end
if result.failed.any? if result.failed.any?
messages << "#{result.failed.size} file(s) failed to upload" failures = result.failed.map { |f| "#{f.filename} (#{f.reason})" }.join("; ")
parts << "Failed: #{failures}"
end end
messages.join(", ") parts.join(". ")
end
if result.failed.any?
failures = result.failed.map { |f| "#{f.filename} (#{f.reason})" }.join(", ")
parts << "Failed: #{failures}"
end
parts.join(". ")
end end
def set_upload def set_upload
@upload = current_user.uploads.find(params[:id]) @upload = Upload.find(params[:id])
end end
end end

View file

@ -5,17 +5,13 @@ require "open-uri"
class Upload < ApplicationRecord class Upload < ApplicationRecord
include PgSearch::Model include PgSearch::Model
# UUID v7 primary key
belongs_to :user belongs_to :user
belongs_to :blob, class_name: "ActiveStorage::Blob" belongs_to :blob, class_name: "ActiveStorage::Blob"
after_destroy :purge_blob after_destroy :purge_blob
# Delegate file metadata to blob
delegate :filename, :byte_size, :content_type, :checksum, to: :blob delegate :filename, :byte_size, :content_type, :checksum, to: :blob
# Search configuration
pg_search_scope :search_by_filename, pg_search_scope :search_by_filename,
associated_against: { associated_against: {
blob: :filename blob: :filename
@ -32,11 +28,9 @@ class Upload < ApplicationRecord
}, },
using: { tsearch: { prefix: true } } using: { tsearch: { prefix: true } }
# Aliases for consistency
alias_method :file_size, :byte_size alias_method :file_size, :byte_size
alias_method :mime_type, :content_type alias_method :mime_type, :content_type
# Provenance enum
enum :provenance, { enum :provenance, {
slack: "slack", slack: "slack",
web: "web", web: "web",
@ -56,13 +50,11 @@ class Upload < ApplicationRecord
ActiveSupport::NumberHelper.number_to_human_size(byte_size) ActiveSupport::NumberHelper.number_to_human_size(byte_size)
end end
# Direct URL to public R2 bucket
def assets_url def assets_url
host = ENV.fetch("CDN_ASSETS_HOST", "cdn.hackclub-assets.com") host = ENV.fetch("CDN_ASSETS_HOST", "cdn.hackclub-assets.com")
"https://#{host}/#{blob.key}" "https://#{host}/#{blob.key}"
end end
# Get CDN URL (uses external uploads controller)
def cdn_url def cdn_url
Rails.application.routes.url_helpers.external_upload_url( Rails.application.routes.url_helpers.external_upload_url(
id:, id:,
@ -71,9 +63,20 @@ class Upload < ApplicationRecord
) )
end end
# Create upload from URL (for API/rescue operations) # Rename the display filename (storage key stays the same)
def rename!(new_filename)
sanitized = ActiveStorage::Filename.new(new_filename).sanitized
# Preserve the original extension if the user didn't provide one
original_ext = File.extname(blob.filename.to_s)
new_ext = File.extname(sanitized)
sanitized = "#{sanitized}#{original_ext}" if new_ext.blank? && original_ext.present?
blob.update!(filename: sanitized)
end
def self.create_from_url(url, user:, provenance:, original_url: nil, authorization: nil, filename: nil) def self.create_from_url(url, user:, provenance:, original_url: nil, authorization: nil, filename: nil)
con = build_http_client conn = build_http_client
headers = {} headers = {}
headers["Authorization"] = authorization if authorization.present? headers["Authorization"] = authorization if authorization.present?
@ -89,9 +92,10 @@ class Upload < ApplicationRecord
filename ||= extract_filename_from_url(url) filename ||= extract_filename_from_url(url)
body = response.body body = response.body
content_type = Marcel::MimeType.for(StringIO.new(body), name: filename) || response.headers["content-type"] || "application/octet-stream" content_type = Marcel::MimeType.for(StringIO.new(body), name: filename) ||
response.headers["content-type"] ||
"application/octet-stream"
# Pre-generate upload ID for predictable storage path
upload_id = SecureRandom.uuid_v7 upload_id = SecureRandom.uuid_v7
sanitized_filename = ActiveStorage::Filename.new(filename).sanitized sanitized_filename = ActiveStorage::Filename.new(filename).sanitized
storage_key = "#{upload_id}/#{sanitized_filename}" storage_key = "#{upload_id}/#{sanitized_filename}"
@ -114,41 +118,43 @@ class Upload < ApplicationRecord
end end
class << self class << self
private private
def build_http_client def build_http_client
Faraday.new(ssl: { verify: true, verify_mod: OpenSSL::SSL::VERIFY_PEER }) do |f| Faraday.new(ssl: { verify: true, verify_mode: OpenSSL::SSL::VERIFY_PEER }) do |f|
f.response Faraday :follow_redirects, limit: 5 f.response :follow_redirects, limit: 5
f.adapter Faraday.default_adapter f.adapter Faraday.default_adapter
end.tap do |conn| end.tap do |conn|
conn.options.open_timeout = 30 conn.options.open_timeout = 30
conn.options.timeout = 120 conn.options.timeout = 120
end end
end
def pre_check_quota_via_head(conn, url, headers, user)
head_response = conn.head(url, nil, headers)
return unless head_response.success?
content_length = head_response.headers["content-length"]&.to_i
return unless content_length && content_length > 0
quota_service = QuotaService.new(user)
policy = quota_service.current_policy
if content_length > policy.max_file_size
raise "File too large: #{ActiveSupport::NumberHelper.number_to_human_size(content_length)} exceeds limit of #{ActiveSupport::NumberHelper.number_to_human_size(policy.max_file_size)}"
end end
endreturn if quota_service.can_upload?(content_length)
raise "File would exceed storage quota" def pre_check_quota_via_head(conn, url, headers, user)
rescue Faraday::Error head_response = conn.head(url, nil, headers)
nil return unless head_response.success?
end
def extract_filename_from_url(url) content_length = head_response.headers["content-length"]&.to_i
File.basename(URI.parse(url).path).presence || "download" return unless content_length && content_length > 0
end
quota_service = QuotaService.new(user)
policy = quota_service.current_policy
if content_length > policy.max_file_size
raise "File too large: #{ActiveSupport::NumberHelper.number_to_human_size(content_length)} " \
"exceeds limit of #{ActiveSupport::NumberHelper.number_to_human_size(policy.max_file_size)}"
end
return if quota_service.can_upload?(content_length)
raise "File would exceed storage quota"
rescue Faraday::Error
nil
end
def extract_filename_from_url(url)
File.basename(URI.parse(url).path).presence || "download"
end
end end
private private

View file

@ -6,6 +6,10 @@ class UploadPolicy < ApplicationPolicy
user.is_admin? || record.user_id == user.id user.is_admin? || record.user_id == user.id
end end
def update?
user.is_admin? || record.user_id == user.id
end
class Scope < ApplicationPolicy::Scope class Scope < ApplicationPolicy::Scope
def resolve def resolve
if user.is_admin? if user.is_admin?

View file

@ -13,7 +13,7 @@ Rails.application.routes.draw do
get "/auth/hack_club/callback", to: "sessions#create" get "/auth/hack_club/callback", to: "sessions#create"
get "/auth/failure", to: "sessions#failure" get "/auth/failure", to: "sessions#failure"
resources :uploads, only: [ :index, :create, :destroy ] resources :uploads, only: [ :index, :create, :update, :destroy ]
resources :api_keys, only: [ :index, :create, :destroy ] resources :api_keys, only: [ :index, :create, :destroy ]
@ -21,7 +21,7 @@ Rails.application.routes.draw do
namespace :v4 do namespace :v4 do
get "me", to: "users#show" get "me", to: "users#show"
post "upload", to: "uploads#create" post "upload", to: "uploads#create"
psot "uploads", to: "uploads#create_batch" post "uploads", to: "uploads#create_batch"
post "upload_from_url", to: "uploads#create_from_url" post "upload_from_url", to: "uploads#create_from_url"
patch "uploads/:id/rename", to: "uploads#rename", as: :upload_rename patch "uploads/:id/rename", to: "uploads#rename", as: :upload_rename
post "revoke", to: "api_keys#revoke" post "revoke", to: "api_keys#revoke"