Add initial sweep of background caching for shared data on page (#157)

* Add initial sweep of background caching for shared data on page

* Rubocop format

* Speed up currently_hacking

* Rubocop format

* Fix active projects job

* Switch to activeprojectsjob for leaderboard

* Remove current project from nav

* Add flamegraph & stackprof for extra profiling

* Silence bullet alerts

* Remove eagerload of users from mini leaderboard

* Comment out expensive flavortext

* Bundle update brakeman

* Remove duplicate stackprof listing

* Add skylight auth to example env

* Add miscomitted background jobs

.env.example

@@ -46,3 +46,5 @@ WILDCARD_HOST=your_wildcard_host_here
 # GitHub oauth used for github signin
 GITHUB_CLIENT_ID=your_github_client_id_here
 GITHUB_CLIENT_SECRET=your_github_client_secret_here
+
+SKYLIGHT_AUTHENTICATION=replace_me

Gemfile

@@ -76,6 +76,9 @@ gem "activerecord-import"
 gem "rack-mini-profiler"
 # For memory profiling via RMP
 gem "memory_profiler"
+gem "flamegraph"
+
+gem "skylight"
 
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem

Gemfile.lock

@@ -105,7 +105,7 @@ GEM
     bindex (0.8.1)
     bootsnap (1.18.4)
       msgpack (~> 1.2)
-    brakeman (7.0.0)
+    brakeman (7.0.2)
       racc
     builder (3.3.0)
     bullet (8.0.2)
@@ -161,6 +161,7 @@ GEM
     ffi-compiler (1.3.2)
       ffi (>= 1.15.5)
       rake
+    flamegraph (0.9.5)
     fugit (1.11.1)
       et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
@@ -404,6 +405,8 @@ GEM
     sentry-ruby (5.23.0)
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
+    skylight (6.0.4)
+      activesupport (>= 5.2.0)
     slack-ruby-client (2.5.2)
       faraday (>= 2.0)
       faraday-mashify
@@ -496,6 +499,7 @@ DEPENDENCIES
   capybara
   debug
   dotenv-rails
+  flamegraph
   good_job
   http
   importmap-rails
@@ -516,6 +520,7 @@ DEPENDENCIES
   selenium-webdriver
   sentry-rails
   sentry-ruby
+  skylight
   slack-ruby-client
   solid_cable
   solid_cache

app/controllers/application_controller.rb

@@ -43,22 +43,6 @@ class ApplicationController < ActionController::Base
   end
 
   def active_users_graph_data
-    # over the last 24 hours, count the number of people who were active each hour
-    hours = Heartbeat.coding_only
-                     .with_valid_timestamps
-                     .where("time > ?", 24.hours.ago.to_f)
-                     .select("(EXTRACT(EPOCH FROM to_timestamp(time))::bigint / 3600 * 3600) as hour, COUNT(DISTINCT user_id) as count")
-                     .group("hour")
-                     .order("hour DESC")
-
-    top_hour_count = hours.max_by(&:count)&.count || 1
-
-    hours = hours.map do |h|
-      {
-        hour: Time.at(h.hour),
-        users: h.count,
-        height: (h.count.to_f / top_hour_count * 100).round
-      }
-    end
+    Cache::ActiveUsersGraphDataJob.perform_now
   end
 end

app/controllers/leaderboards_controller.rb

@@ -24,7 +24,7 @@ class LeaderboardsController < ApplicationController
     else
       # Load entries with users and their project repo mappings in a single query
       @entries = @leaderboard.entries
-                             .includes(user: :project_repo_mappings)
+                             .includes(:user)
                              .order(total_seconds: :desc)
 
       tracked_user_ids = @leaderboard.entries.distinct.pluck(:user_id)
@@ -47,24 +47,7 @@ class LeaderboardsController < ApplicationController
             .count { |user_id| !tracked_user_ids.include?(user_id) }
       end
 
-      # Get active projects for the leaderboard entries
-      if @entries&.any?
-        user_ids = @entries.pluck(:user_id)
-
-        # Use the faster DISTINCT ON approach for heartbeats
-        # This query gets the most recent heartbeat for each user in a single efficient query
-        recent_heartbeats = Heartbeat.where(user_id: user_ids, source_type: :direct_entry)
-                                   .select("DISTINCT ON (user_id) user_id, project, time")
-                                   .order("user_id, time DESC")
-                                   .index_by(&:user_id)
-
-        @active_projects = {}
-        @entries.each do |entry|
-          user = entry.user
-          recent_heartbeat = recent_heartbeats[user.id]
-          @active_projects[user.id] = user.project_repo_mappings.find { |p| p.project_name == recent_heartbeat&.project }
-        end
-      end
+      @active_projects = Cache::ActiveProjectsJob.perform_now
     end
   end
 end

app/controllers/static_pages_controller.rb

@@ -7,6 +7,9 @@ class StaticPagesController < ApplicationController
                               .distinct
                               .first
 
+    # Get active projects for the mini leaderboard
+    @active_projects = Cache::ActiveProjectsJob.perform_now
+
     if current_user
       flavor_texts = FlavorText.motto + FlavorText.conditional_mottos(current_user)
       flavor_texts += FlavorText.rare_motto if Random.rand(10) < 1
@@ -71,18 +74,7 @@ class StaticPagesController < ApplicationController
         end
       end
 
-      @home_stats = Rails.cache.read("home_stats")
-      CacheHomeStatsJob.perform_later if @home_stats.nil?
-    end
-
-    # Get active projects for the mini leaderboard
-    if @leaderboard
-      user_ids = @leaderboard.entries.pluck(:user_id)
-      users = User.where(id: user_ids).includes(:project_repo_mappings)
-      @active_projects = {}
-      users.each do |user|
-        @active_projects[user.id] = user.project_repo_mappings.find { |p| p.project_name == user.active_project }
-      end
+      @home_stats = Cache::HomeStatsJob.perform_now
     end
   end
 
@@ -127,31 +119,7 @@ class StaticPagesController < ApplicationController
   end
 
   def currently_hacking
-    # Get all users who have heartbeats in the last 15 minutes
-    locals = Rails.cache.fetch("currently_hacking", expires_in: 10.seconds) do
-      user_ids = Heartbeat.where("time > ?", 5.minutes.ago.to_f)
-                          .coding_only
-                          .distinct
-                          .pluck(:user_id)
-
-      users = User.where(id: user_ids).includes(:project_repo_mappings)
-
-      active_projects = {}
-      users.each do |user|
-        active_projects[user.id] = user.project_repo_mappings.find { |p| p.project_name == user.active_project }
-      end
-
-      users = users.sort_by do |user|
-        [
-          active_projects[user.id].present? ? 0 : 1,
-          user.username.present? ? 0 : 1,
-          user.slack_username.present? ? 0 : 1,
-          user.github_username.present? ? 0 : 1
-        ]
-      end
-
-      { users: users, active_projects: active_projects }
-    end
+    locals = Cache::CurrentlyHackingJob.perform_now
 
     render partial: "currently_hacking", locals: locals
   end

app/jobs/cache/active_projects_job.rb

@@ -0,0 +1,32 @@
+class Cache::ActiveProjectsJob < ApplicationJob
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  # Limits concurrency to 1 job per date
+  good_job_control_concurrency_with(
+    total: 1,
+    drop: true
+  )
+
+  def perform(force_reload: false)
+    key = "active_projects"
+    expiration = 15.minutes
+    Rails.cache.write(key, calculate, expires_in: expiration) if force_reload
+
+    Rails.cache.fetch(key, expires_in: expiration,) do
+      calculate
+    end
+  end
+
+  private
+
+  def calculate
+    # Get recent heartbeats with matching project_repo_mappings in a single SQL query
+    ProjectRepoMapping.joins("INNER JOIN heartbeats ON heartbeats.project = project_repo_mappings.project_name")
+                      .joins("INNER JOIN users ON users.id = heartbeats.user_id")
+                      .where("heartbeats.source_type = ?", Heartbeat.source_types[:direct_entry])
+                      .where("heartbeats.time > ?", 5.minutes.ago.to_f)
+                      .select("DISTINCT ON (heartbeats.user_id) project_repo_mappings.*, heartbeats.user_id")
+                      .order("heartbeats.user_id, heartbeats.time DESC")
+                      .index_by(&:user_id)
+  end
+end

app/jobs/cache/active_users_graph_data_job.rb

@@ -0,0 +1,42 @@
+class Cache::ActiveUsersGraphDataJob < ApplicationJob
+  # TODO: create concern for these cache jobs– make it single enqueue
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  # Limits concurrency to 1 job per date
+  good_job_control_concurrency_with(
+    total: 1,
+    drop: true
+  )
+
+  def perform(force_reload: false)
+    key = "cache:active_users_graph_data"
+    expiration = 1.hour
+    Rails.cache.write(key, calculate, expires_in: expiration) if force_reload
+
+    Rails.cache.fetch(key, expires_in: expiration) do
+      calculate
+    end
+  end
+
+  private
+
+  def calculate
+    # over the last 24 hours, count the number of people who were active each hour
+    hours = Heartbeat.coding_only
+                     .with_valid_timestamps
+                     .where("time > ?", 24.hours.ago.to_f)
+                     .select("(EXTRACT(EPOCH FROM to_timestamp(time))::bigint / 3600 * 3600) as hour, COUNT(DISTINCT user_id) as count")
+                     .group("hour")
+                     .order("hour DESC")
+
+    top_hour_count = hours.max_by(&:count)&.count || 1
+
+    hours = hours.map do |h|
+      {
+        hour: Time.at(h.hour),
+        users: h.count,
+        height: (h.count.to_f / top_hour_count * 100).round
+      }
+    end
+  end
+end

app/jobs/cache/currently_hacking_job.rb

@@ -0,0 +1,52 @@
+class Cache::CurrentlyHackingJob < ApplicationJob
+  include GoodJob::ActiveJobExtensions::Concurrency
+
+  # Limits concurrency to 1 job per date
+  good_job_control_concurrency_with(
+    total: 1,
+    drop: true
+  )
+
+  def perform(force_reload: false)
+    key = "currently_hacking"
+    expiration = 1.minute
+    Rails.cache.write(key, calculate, expires_in: expiration) if force_reload
+
+    Rails.cache.fetch(key, expires_in: expiration) do
+      calculate
+    end
+  end
+
+  private
+
+  def calculate
+    # Get most recent heartbeats and users in a single query
+    recent_heartbeats = Heartbeat.joins(:user)
+                                .where(source_type: :direct_entry)
+                                .coding_only
+                                .where("time > ?", 5.minutes.ago.to_f)
+                                .select("DISTINCT ON (user_id) user_id, project, time, users.*")
+                                .order("user_id, time DESC")
+                                .includes(user: :project_repo_mappings)
+                                .index_by(&:user_id)
+
+    users = recent_heartbeats.values.map(&:user)
+
+    active_projects = {}
+    users.each do |user|
+      recent_heartbeat = recent_heartbeats[user.id]
+      active_projects[user.id] = user.project_repo_mappings.find { |p| p.project_name == recent_heartbeat&.project }
+    end
+
+    users = users.sort_by do |user|
+      [
+        active_projects[user.id].present? ? 0 : 1,
+        user.username.present? ? 0 : 1,
+        user.slack_username.present? ? 0 : 1,
+        user.github_username.present? ? 0 : 1
+      ]
+    end
+
+    { users: users, active_projects: active_projects }
+  end
+end

app/jobs/cache/home_stats_job.rb

@@ -1,4 +1,4 @@
-class CacheHomeStatsJob < ApplicationJob
+class Cache::HomeStatsJob < ApplicationJob
   include GoodJob::ActiveJobExtensions::Concurrency
 
   # Limits concurrency to 1 job per date
@@ -7,13 +7,23 @@ class CacheHomeStatsJob < ApplicationJob
     drop: true
   )
 
-  def perform
+  def perform(force_reload: false)
+    key = "home_stats"
+    expiration = 1.hour
+    Rails.cache.write(key, calculate, expires_in: expiration) if force_reload
+
+    Rails.cache.fetch(key, expires_in: expiration,) do
+      calculate
+    end
+  end
+
+  private
+
+  def calculate
     seconds_by_user = Heartbeat.group(:user_id).duration_seconds
-    home_stats = {
+    {
       users_tracked: seconds_by_user.size,
       seconds_tracked: seconds_by_user.values.sum
     }
-    Rails.cache.write("home_stats", home_stats, expires_in: 1.hour)
-    home_stats
   end
 end

app/models/user.rb

@@ -312,20 +312,6 @@ class User < ApplicationRecord
     email.split("@")&.first.truncate(10) + " (email sign-up)"
   end
 
-  def project_names
-    heartbeats.select(:project).distinct.pluck(:project)
-  end
-
-  def active_project
-    most_recent_direct_entry_heartbeat&.project
-  end
-
-  def active_project_duration
-    return nil unless active_project
-
-    heartbeats.where(project: active_project).duration_seconds
-  end
-
   def most_recent_direct_entry_heartbeat
     heartbeats.where(source_type: :direct_entry).order(time: :desc).first
   end

app/views/leaderboards/_mini_leaderboard.html.erb

@@ -1,5 +1,5 @@
 <%
-  entries = leaderboard.entries.includes(:user).order(total_seconds: :desc)
+  entries = leaderboard.entries.order(total_seconds: :desc)
   if current_user
     user_rank = entries.find_index { |entry| entry.user_id == current_user.id }
     if user_rank && user_rank >= 3

app/views/shared/_nav.html.erb

@@ -15,12 +15,6 @@
           <%= current_user.streak_days_formatted %> 🔥
         </p>
       <% end %>
-      <% if current_user.active_project && current_user.active_project_duration > 60 %>
-        <p>
-          Working on: <%= current_user.active_project %>
-          (<%= short_time_simple current_user.active_project_duration %> in total)
-        </p>
-      <% end %>
     </li>
   <% else %>
     <li>

config/environments/development.rb

@@ -3,7 +3,7 @@ require "active_support/core_ext/integer/time"
 Rails.application.configure do
   config.after_initialize do
     Bullet.enable        = true
-    Bullet.alert         = true
+    Bullet.alert         = false
     Bullet.bullet_logger = true
     Bullet.console       = true
     Bullet.rails_logger  = true

config/initializers/good_job.rb

@@ -35,13 +35,29 @@ Rails.application.configure do
       cron: "0 0 * * *",
       class: "SlackUsernameUpdateJob"
     },
-    cache_home_stats: {
-      cron: "0/10 * * * *",
-      class: "CacheHomeStatsJob"
-    },
     scan_github_repos: {
       cron: "0 10 * * *",
       class: "ScanGithubReposJob"
+    },
+    cache_active_user_graph_data_job: {
+      cron: "*/10 * * * *",
+      class: "Cache::ActiveUsersGraphDataJob",
+      args: [ dry_run: false ]
+    },
+    cache_currently_hacking: {
+      cron: "* * * * *",
+      class: "Cache::CurrentlyHacking",
+      args: [ dry_run: false ]
+    },
+    cache_home_stats: {
+      cron: "*/10 * * * *",
+      class: "Cache::HomeStatsJob",
+      args: [ dry_run: false ]
+    },
+    cache_active_projects: {
+      cron: "* * * * *",
+      class: "Cache::ActiveProjectsJob",
+      args: [ dry_run: false ]
     }
   }
 end

lib/flavor_text.rb

@@ -228,8 +228,8 @@ class FlavorText
     r << "in the nick of time!" if %w[nick nicholas nickolas].include?(user.username)
     r << "just-in time!" if %w[justin justine].include?(user.username)
 
-    minutes_logged = Heartbeat.where("time > ?", 1.hour.ago.to_f).duration_seconds / 60
-    r << "in the past hour, #{minutes_logged} minutes have passed" if minutes_logged > 0
+    # minutes_logged = -> Heartbeat.where("time > ?", 1.hour.ago.to_f..Time.current.to_f).duration_seconds / 60
+    # r << "in the past hour, #{minutes_logged} minutes have passed" if minutes_logged > 0
 
     r
   end