Attempt to sync request counter between processes

This commit is contained in:
Max Wofford 2025-06-27 22:41:35 -04:00
parent bcc5b03dd5
commit 25dfaffb21
2 changed files with 74 additions and 6 deletions

View file

@ -10,6 +10,11 @@ module ApplicationHelper
rps == :high_load ? "lots of req/sec" : "#{rps} req/sec"
end
def global_requests_per_second
rps = RequestCounter.global_per_second
rps == :high_load ? "lots of req/sec" : "#{rps} req/sec (global)"
end
def admin_tool(class_name = "", element = "div", **options, &block)
return unless current_user&.is_admin?
concat content_tag(element, class: "admin-tool #{class_name}", **options, &block)

View file

@ -2,9 +2,12 @@ class RequestCounter
WINDOW_SIZE = 10 # seconds - shorter window for more responsive rates
HIGH_LOAD_THRESHOLD = 500 # req/sec to disable tracking
CIRCUIT_BREAKER_DURATION = 30 # seconds to stay disabled
PROCESS_ID = "#{Socket.gethostname}-#{Process.pid}"
STATS_DIR = Rails.root.join("tmp", "request_stats")
@buckets = {}
@disabled_until = nil
@last_sync = 0
class << self
def increment
@ -16,8 +19,11 @@ class RequestCounter
# Check if we should disable due to high load
check_circuit_breaker(current_time)
# Periodically clean old buckets (1% chance)
cleanup if rand(100) == 0
# Periodically sync to file and cleanup (1% chance)
if rand(100) == 0
sync_to_file(current_time)
cleanup
end
end
def per_second
@ -26,7 +32,36 @@ class RequestCounter
current_time = Time.current.to_i
cutoff = current_time - WINDOW_SIZE
total = @buckets.select { |timestamp, _| timestamp >= cutoff }.values.sum
# Fast local calculation
local_total = @buckets.select { |timestamp, _| timestamp >= cutoff }.values.sum
(local_total.to_f / WINDOW_SIZE).round(2)
end
def global_per_second
return :high_load if disabled?
current_time = Time.current.to_i
sync_to_file(current_time)
# Read and aggregate from all process files
cutoff = current_time - WINDOW_SIZE
total = 0
Dir.glob(STATS_DIR.join("*.txt")).each do |file_path|
next unless File.mtime(file_path) > (cutoff - 60).seconds.ago # Skip very old files
begin
File.read(file_path).each_line do |line|
next if line.strip.empty?
timestamp, count = line.strip.split(":", 2)
next unless timestamp && count
total += count.to_i if timestamp.to_i >= cutoff
end
rescue Errno::ENOENT
# Skip deleted files
end
end
(total.to_f / WINDOW_SIZE).round(2)
end
@ -37,7 +72,7 @@ class RequestCounter
end
def check_circuit_breaker(current_time)
# Check last 5 seconds for high load
# Check last 5 seconds for high load (local only for performance)
recent_total = @buckets.select { |ts, _| ts >= current_time - 5 }.values.sum
if recent_total > HIGH_LOAD_THRESHOLD * 5 # 5 seconds worth
@ -46,12 +81,40 @@ class RequestCounter
end
end
def cleanup
return if disabled? # Skip cleanup when disabled
def sync_to_file(current_time)
return if current_time == @last_sync || @buckets.empty?
ensure_stats_dir
file_path = STATS_DIR.join("#{PROCESS_ID}.txt")
# Atomic write: write to temp file then rename
temp_path = "#{file_path}.tmp"
data = @buckets.map { |timestamp, count| "#{timestamp}:#{count}" }.join("\n")
File.write(temp_path, data)
File.rename(temp_path, file_path)
@last_sync = current_time
rescue Errno::ENOENT, Errno::EACCES
# Silently fail if we can't write (e.g., read-only filesystem)
end
def ensure_stats_dir
FileUtils.mkdir_p(STATS_DIR) unless Dir.exist?(STATS_DIR)
end
def cleanup
current_time = Time.current.to_i
cutoff = current_time - WINDOW_SIZE - 10 # extra buffer
@buckets.reject! { |timestamp, _| timestamp < cutoff }
# Clean up old process files (10% chance)
return unless rand(10) == 0
Dir.glob(STATS_DIR.join("*.txt")).each do |file_path|
File.delete(file_path) if File.mtime(file_path) < (cutoff - 60).seconds.ago
rescue Errno::ENOENT
# File already deleted
end
end
end
end