profile speedups and cache (#811)

app/controllers/profiles_controller.rb

@@ -23,49 +23,15 @@ class ProfilesController < ApplicationController
 
   def load
     Time.use_zone(@user.timezone) do
-      @total_time_today = @user.heartbeats.today.duration_seconds
-      @total_time_week = @user.heartbeats.this_week.duration_seconds
-      @total_time_all = @user.heartbeats.duration_seconds
+      stats = ProfileStatsService.new(@user).stats
 
-      @top_languages = @user.heartbeats
-        .where.not(language: [ nil, "" ])
-        .group(:language)
-        .duration_seconds
-        .sort_by { |_, v| -v }
-        .first(5)
-        .to_h
-
-      @top_projects = @user.heartbeats
-        .group(:project)
-        .duration_seconds
-        .sort_by { |_, v| -v }
-        .first(5)
-        .to_h
-
-      project_repo_mappings = @user.project_repo_mappings.active.index_by(&:project_name)
-
-      @top_projects_month = @user.heartbeats
-        .where("time >= ?", 1.month.ago.to_f)
-        .group(:project)
-        .duration_seconds
-        .sort_by { |_, v| -v }
-        .first(6)
-        .map do |project, duration|
-          mapping = project_repo_mappings[project]
-          { project: project, duration: duration, repo_url: mapping&.repo_url }
-        end
-
-      @top_editors = @user.heartbeats
-        .where.not(editor: [ nil, "" ])
-        .group(:editor)
-        .duration_seconds
-        .each_with_object(Hash.new(0)) do |(editor, duration), acc|
-          normalized = ApplicationController.helpers.display_editor_name(editor)
-          acc[normalized] += duration
-        end
-        .sort_by { |_, v| -v }
-        .first(3)
-        .to_h
+      @total_time_today = stats[:total_time_today]
+      @total_time_week = stats[:total_time_week]
+      @total_time_all = stats[:total_time_all]
+      @top_languages = stats[:top_languages]
+      @top_projects = stats[:top_projects]
+      @top_projects_month = stats[:top_projects_month]
+      @top_editors = stats[:top_editors]
 
       @daily_durations = @user.heartbeats.daily_durations(user_timezone: @user.timezone).to_h
 

app/models/concerns/heartbeatable.rb

@@ -109,6 +109,7 @@ module Heartbeatable
         return user_ids.index_with { |id| streak_cache["user_streak_#{id}"] || 0 }
       end
 
+      timeout = heartbeat_timeout_duration.to_i
       raw_durations = joins(:user)
         .where(user_id: uncached_users)
         .coding_only
@@ -118,7 +119,7 @@ module Heartbeatable
           :user_id,
           "users.timezone as user_timezone",
           Arel.sql("DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone) as day_group"),
-          Arel.sql("LEAST(EXTRACT(EPOCH FROM (to_timestamp(time) - to_timestamp(LAG(time) OVER (PARTITION BY user_id, DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone) ORDER BY time)))), #{heartbeat_timeout_duration.to_i}) as diff")
+          Arel.sql("LEAST(time - LAG(time) OVER (PARTITION BY user_id, DATE_TRUNC('day', to_timestamp(time) AT TIME ZONE users.timezone) ORDER BY time), #{timeout}) as diff")
         )
 
       # Then aggregate the results
@@ -208,6 +209,7 @@ module Heartbeatable
 
     def duration_seconds(scope = all)
       scope = scope.with_valid_timestamps
+      timeout = heartbeat_timeout_duration.to_i
 
       if scope.group_values.any?
         if scope.group_values.length > 1
@@ -222,7 +224,7 @@ module Heartbeatable
         capped_diffs = scope
           .select("#{group_expr} as grouped_time, CASE
             WHEN LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time) IS NULL THEN 0
-            ELSE LEAST(EXTRACT(EPOCH FROM (to_timestamp(time) - to_timestamp(LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time)))), #{heartbeat_timeout_duration.to_i})
+            ELSE LEAST(time - LAG(time) OVER (PARTITION BY #{group_expr} ORDER BY time), #{timeout})
           END as diff")
           .where.not(time: nil)
           .unscope(:group)
@@ -239,7 +241,7 @@ module Heartbeatable
         capped_diffs = scope
           .select("CASE
             WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
-            ELSE LEAST(EXTRACT(EPOCH FROM (to_timestamp(time) - to_timestamp(LAG(time) OVER (ORDER BY time)))), #{heartbeat_timeout_duration.to_i})
+            ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout})
           END as diff")
           .where.not(time: nil)
 
@@ -290,10 +292,11 @@ module Heartbeatable
       end
 
       # we calc w/ the boundary heartbeat, but we only sum within the orignal constraint
+      timeout = heartbeat_timeout_duration.to_i
       capped_diffs = combined_scope
         .select("time, CASE
           WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
-          ELSE LEAST(EXTRACT(EPOCH FROM (to_timestamp(time) - to_timestamp(LAG(time) OVER (ORDER BY time)))), #{heartbeat_timeout_duration.to_i})
+          ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout})
         END as diff")
         .where.not(time: nil)
         .order(time: :asc)

app/services/profile_stats_service.rb

@@ -0,0 +1,157 @@
+class ProfileStatsService
+  CACHE_TTL = 5.minutes
+
+  attr_reader :user
+
+  def initialize(user)
+    @user = user
+  end
+
+  def stats
+    Rails.cache.fetch(cache_key, expires_in: CACHE_TTL) do
+      compute_stats
+    end
+  end
+
+  private
+
+  def cache_key
+    latest_heartbeat_time = user.heartbeats.maximum(:time) || 0
+    "profile_stats:v2:user:#{user.id}:latest:#{latest_heartbeat_time}"
+  end
+
+  def compute_stats
+    Time.use_zone(user.timezone) do
+      timeout = Heartbeat.heartbeat_timeout_duration.to_i
+
+      today_start = Time.current.beginning_of_day.to_i
+      today_end = Time.current.end_of_day.to_i
+      week_start = Time.current.beginning_of_week.to_i
+      week_end = Time.current.end_of_week.to_i
+      month_ago = 1.month.ago.to_f
+
+      base_result = compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, week_end, month_ago)
+
+      {
+        total_time_today: base_result[:today_seconds],
+        total_time_week: base_result[:week_seconds],
+        total_time_all: base_result[:all_seconds],
+        top_languages: base_result[:top_languages],
+        top_projects: base_result[:top_projects],
+        top_projects_month: base_result[:top_projects_month],
+        top_editors: base_result[:top_editors]
+      }
+    end
+  end
+
+  def compute_totals_and_breakdowns(timeout, today_start, today_end, week_start, week_end, month_ago)
+    conn = Heartbeat.connection
+    user_id = conn.quote(user.id)
+    timeout_quoted = conn.quote(timeout)
+    today_start_quoted = conn.quote(today_start)
+    today_end_quoted = conn.quote(today_end)
+    week_start_quoted = conn.quote(week_start)
+    week_end_quoted = conn.quote(week_end)
+    month_ago_quoted = conn.quote(month_ago)
+
+    base_sql = <<~SQL
+      WITH heartbeat_diffs AS (
+        SELECT
+          time,
+          project,
+          language,
+          editor,
+          CASE
+            WHEN LAG(time) OVER (ORDER BY time) IS NULL THEN 0
+            ELSE LEAST(time - LAG(time) OVER (ORDER BY time), #{timeout_quoted})
+          END AS diff
+        FROM heartbeats
+        WHERE user_id = #{user_id}
+          AND deleted_at IS NULL
+          AND time IS NOT NULL
+          AND time >= 0 AND time <= 253402300799
+      )
+    SQL
+
+    totals_sql = <<~SQL
+      #{base_sql}
+      SELECT
+        COALESCE(SUM(diff) FILTER (WHERE time >= #{today_start_quoted} AND time <= #{today_end_quoted}), 0)::integer AS today_seconds,
+        COALESCE(SUM(diff) FILTER (WHERE time >= #{week_start_quoted} AND time <= #{week_end_quoted}), 0)::integer AS week_seconds,
+        COALESCE(SUM(diff), 0)::integer AS all_seconds
+      FROM heartbeat_diffs
+    SQL
+
+    totals = conn.select_one(totals_sql)
+
+    top_languages = fetch_top_grouped(conn, base_sql, "language", nil, 5)
+    top_projects_all = fetch_top_grouped(conn, base_sql, "project", nil, 5)
+    top_projects_month = fetch_top_grouped_with_repo(conn, base_sql, month_ago_quoted, 6)
+    top_editors = fetch_top_editors_normalized(conn, base_sql, 3)
+
+    {
+      today_seconds: totals["today_seconds"].to_i,
+      week_seconds: totals["week_seconds"].to_i,
+      all_seconds: totals["all_seconds"].to_i,
+      top_languages: top_languages,
+      top_projects: top_projects_all,
+      top_projects_month: top_projects_month,
+      top_editors: top_editors
+    }
+  end
+
+  def fetch_top_grouped(conn, base_sql, column, time_filter, limit)
+    time_clause = time_filter ? "AND time >= #{time_filter}" : ""
+    sql = <<~SQL
+      #{base_sql}
+      SELECT #{column}, COALESCE(SUM(diff), 0)::integer AS duration
+      FROM heartbeat_diffs
+      WHERE #{column} IS NOT NULL AND #{column} != ''
+      #{time_clause}
+      GROUP BY #{column}
+      ORDER BY duration DESC
+      LIMIT #{limit}
+    SQL
+
+    conn.select_all(sql).each_with_object({}) do |row, hash|
+      hash[row[column]] = row["duration"].to_i
+    end
+  end
+
+  def fetch_top_grouped_with_repo(conn, base_sql, month_ago, limit)
+    sql = <<~SQL
+      #{base_sql}
+      SELECT project, COALESCE(SUM(diff), 0)::integer AS duration
+      FROM heartbeat_diffs
+      WHERE time >= #{month_ago}
+        AND project IS NOT NULL AND project != ''
+      GROUP BY project
+      ORDER BY duration DESC
+      LIMIT #{limit}
+    SQL
+
+    project_repo_mappings = user.project_repo_mappings.active.index_by(&:project_name)
+
+    conn.select_all(sql).map do |row|
+      project = row["project"]
+      mapping = project_repo_mappings[project]
+      { project: project, duration: row["duration"].to_i, repo_url: mapping&.repo_url }
+    end
+  end
+
+  def fetch_top_editors_normalized(conn, base_sql, limit)
+    sql = <<~SQL
+      #{base_sql}
+      SELECT editor, COALESCE(SUM(diff), 0)::integer AS duration
+      FROM heartbeat_diffs
+      WHERE editor IS NOT NULL AND editor != ''
+      GROUP BY editor
+      ORDER BY duration DESC
+    SQL
+
+    conn.select_all(sql).each_with_object(Hash.new(0)) do |row, acc|
+      normalized = ApplicationController.helpers.display_editor_name(row["editor"])
+      acc[normalized] += row["duration"].to_i
+    end.sort_by { |_, v| -v }.first(limit).to_h
+  end
+end