Files
coder/coderd/database/queries/insights.sql
T
Ehab Younes 817fc77612 feat: store session usage per app and family in template usage stats (#29109)
Store per-app session minutes in a child table instead of fixed family-minute columns, so adding an IDE no longer needs a rollup schema change. API, Prometheus, and telemetry output are unchanged.

- Add `template_usage_stats_session_apps`, keyed by start time, template, user, and app name, replacing the `ssh_mins`, `sftp_mins`, `reconnecting_pty_mins`, `vscode_mins`, and `jetbrains_mins` columns.
- Group app names into families at read time through the `codersdk` registry, so the registry never reaches SQL and attribution covers all history rather than only the buckets the rollup still revisits.
- Group `GetTemplateInsightsByTemplate` by app name so that Prometheus and the API report a family identically.
- Drop the `@app_families` query parameter and its dbauthz validation.
- Report a family total as the sum of its apps rather than the distinct minutes any of them was active. Numbers are unchanged today, because every agent reports one canonical name per family.
- Keep the child table window index-usable and count minutes with `COUNT(DISTINCT)` in the grouping, so a dashboard-sized range uses an index scan and a large deployment does not spill the sort to disk.

Part of #27410
2026-09-18 15:26:47 +02:00

1030 lines
32 KiB
SQL

-- name: GetUserLatencyInsights :many
-- GetUserLatencyInsights returns the median and 95th percentile connection
-- latency that users have experienced. The result can be filtered on
-- template_ids, meaning only user data from workspaces based on those templates
-- will be included.
SELECT
tus.user_id,
u.username,
u.avatar_url,
array_agg(DISTINCT tus.template_id)::uuid[] AS template_ids,
COALESCE((PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY tus.median_latency_ms)), -1)::float AS workspace_connection_latency_50,
COALESCE((PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY tus.median_latency_ms)), -1)::float AS workspace_connection_latency_95
FROM
template_usage_stats tus
JOIN
users u
ON
u.id = tus.user_id
WHERE
tus.start_time >= @start_time::timestamptz
AND tus.end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN tus.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
tus.user_id, u.username, u.avatar_url
ORDER BY
tus.user_id ASC;
-- name: GetUserActivityInsights :many
-- GetUserActivityInsights returns the ranking with top active users.
-- The result can be filtered on template_ids, meaning only user data
-- from workspaces based on those templates will be included.
-- Note: The usage_seconds and usage_seconds_cumulative differ only when
-- requesting deployment-wide (or multiple template) data. Cumulative
-- produces a bloated value if a user has used multiple templates
-- simultaneously.
WITH
deployment_stats AS (
SELECT
start_time,
user_id,
array_agg(template_id) AS template_ids,
-- See motivation in GetTemplateInsights for LEAST(SUM(n), 30).
LEAST(SUM(usage_mins), 30) AS usage_mins
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
start_time, user_id
),
template_ids AS (
SELECT
user_id,
array_agg(DISTINCT template_id) AS ids
FROM
deployment_stats, unnest(template_ids) template_id
GROUP BY
user_id
)
SELECT
ds.user_id,
u.username,
u.avatar_url,
t.ids::uuid[] AS template_ids,
(SUM(ds.usage_mins) * 60)::bigint AS usage_seconds
FROM
deployment_stats ds
JOIN
users u
ON
u.id = ds.user_id
JOIN
template_ids t
ON
ds.user_id = t.user_id
GROUP BY
ds.user_id, u.username, u.avatar_url, t.ids
ORDER BY
ds.user_id ASC;
-- name: GetTemplateInsights :one
-- GetTemplateInsights returns the aggregate user-produced usage of all
-- workspaces in a given timeframe. The template IDs, active users, and
-- usage_seconds all reflect any usage in the template, including apps.
--
-- Session usage comes out per app name; callers group the names into families.
--
-- When combining data from multiple templates, we must make a guess at
-- how the user behaved for the 30 minute interval. In this case we make
-- the assumption that if the user used two workspaces for 15 minutes,
-- they did so sequentially, thus we sum the usage up to a maximum of
-- 30 minutes with LEAST(SUM(n), 30).
WITH
base AS (
-- One pass over the window: per-user capped minutes and the template
-- list. GROUPING tells the two row kinds apart.
SELECT
GROUPING(template_id) = 1 AS is_user_row,
start_time,
user_id,
template_id,
COUNT(*) AS template_count,
LEAST(SUM(usage_mins), 30) AS usage_mins
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY GROUPING SETS ((start_time, user_id), (template_id))
),
users AS (
SELECT
start_time,
user_id,
template_count,
usage_mins
FROM
base
WHERE
is_user_row
),
multi_template_buckets AS (
-- An app's minutes cap per user per half hour, across templates. Only
-- these buckets can reach the cap, and most deployments have few, so
-- the capped grouping below runs on them alone.
SELECT
start_time,
user_id
FROM
users
WHERE
template_count > 1
),
app_usage_by_template AS (
-- A single row cannot exceed the cap, so these need no per-user
-- grouping. FILTER, not WHERE: the excluded buckets' minutes belong to
-- app_usage_capped, but their templates still belong in the list.
SELECT
sessions.app_name,
sessions.template_id,
SUM(sessions.usage_mins) FILTER (
WHERE (sessions.start_time, sessions.user_id) NOT IN (SELECT start_time, user_id FROM multi_template_buckets)
) AS usage_mins
FROM
template_usage_stats_session_apps AS sessions
WHERE
sessions.start_time >= @start_time::timestamptz
-- The child table has no end_time, hence the bucket width. Keep
-- start_time bare so the range stays index-usable.
AND sessions.start_time <= (@end_time::timestamptz) - '30 minutes'::interval
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
sessions.app_name, sessions.template_id
),
app_usage_capped AS (
SELECT
sessions.app_name,
LEAST(SUM(sessions.usage_mins), 30) AS usage_mins
FROM
template_usage_stats_session_apps AS sessions
WHERE
sessions.start_time >= @start_time::timestamptz
AND sessions.start_time <= (@end_time::timestamptz) - '30 minutes'::interval
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN sessions.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
AND EXISTS (
SELECT 1
FROM multi_template_buckets AS buckets
WHERE buckets.start_time = sessions.start_time
AND buckets.user_id = sessions.user_id
)
GROUP BY
sessions.start_time, sessions.user_id, sessions.app_name
),
app_usage AS (
SELECT
app_name,
(SUM(usage_mins) * 60)::bigint AS usage_seconds
FROM (
SELECT app_name, usage_mins FROM app_usage_by_template
UNION ALL
SELECT app_name, usage_mins FROM app_usage_capped
) AS parts
GROUP BY
app_name
),
app_templates AS (
SELECT
app_name,
array_agg(DISTINCT template_id) AS template_ids
FROM
app_usage_by_template
GROUP BY
app_name
)
SELECT
COALESCE((SELECT array_agg(DISTINCT template_id) FROM base WHERE NOT is_user_row), '{}')::uuid[] AS template_ids, -- Includes app usage.
COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage.
COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage.
-- Keyed by app name; callers fold both into families.
COALESCE((SELECT jsonb_object_agg(app_name, usage_seconds) FROM app_usage), '{}'::jsonb)::jsonb AS session_app_usage_seconds,
COALESCE((SELECT jsonb_object_agg(app_name, template_ids) FROM app_templates), '{}'::jsonb)::jsonb AS session_app_template_ids
FROM
users;
-- name: GetTemplateInsightsByTemplate :many
-- GetTemplateInsightsByTemplate is used for Prometheus metrics. Keep
-- in sync with GetTemplateInsights and UpsertTemplateUsageStats.
--
-- Session usage comes out per app name, as in GetTemplateInsights, so either
-- query reports the same family totals once the names are grouped.
WITH
connected AS (
-- NOTE(mafredri): connection_count covers one report interval, while
-- the session counts are a gauge, so an idle session reports none.
-- Hence "any connection within this bucket", pending a better solution.
-- Grouped, not a WHERE: one connection anywhere in the window keeps
-- every minute of that pair. One row per pair, so no fan-out.
SELECT
template_id,
user_id
FROM
workspace_agent_stats
WHERE
created_at >= @start_time::timestamptz
AND created_at < @end_time::timestamptz
AND session_counts <> '{}'::jsonb
GROUP BY
template_id, user_id
HAVING
BOOL_OR(connection_count > 0)
),
insights AS (
-- A minute counts once per app however many of its sessions were open,
-- which COUNT(DISTINCT) does in the grouping. Deduplicating the
-- expanded rows first instead spills to disk once a deployment has a
-- few thousand agents.
SELECT
was.template_id,
was.user_id,
app_name,
COUNT(DISTINCT date_trunc('minute', was.created_at)) AS usage_mins
FROM
workspace_agent_stats AS was
CROSS JOIN
jsonb_object_keys(was.session_counts) AS app_name
JOIN
connected AS c
ON
c.template_id = was.template_id
AND c.user_id = was.user_id
WHERE
was.created_at >= @start_time::timestamptz
AND was.created_at < @end_time::timestamptz
AND was.session_counts <> '{}'::jsonb
GROUP BY
was.template_id, was.user_id, app_name
),
app_usage AS (
SELECT
template_id,
jsonb_object_agg(app_name, usage_seconds) AS session_app_usage_seconds
FROM (
SELECT
template_id,
app_name,
(SUM(usage_mins) * 60)::bigint AS usage_seconds
FROM
insights
GROUP BY
template_id, app_name
) AS app_seconds
GROUP BY
template_id
),
active_users AS (
SELECT
template_id,
COUNT(DISTINCT user_id)::bigint AS active_users
FROM
insights
GROUP BY
template_id
)
SELECT
active_users.template_id,
active_users.active_users,
app_usage.session_app_usage_seconds
FROM
active_users
JOIN
-- Every counted template has at least one app; both sides come from
-- insights.
app_usage
ON
app_usage.template_id = active_users.template_id;
-- name: GetTemplateAppInsights :many
-- GetTemplateAppInsights returns the aggregate usage of each app in a given
-- timeframe. The result can be filtered on template_ids, meaning only user data
-- from workspaces based on those templates will be included.
WITH
-- Create a list of all unique apps by template, this is used to
-- filter out irrelevant template usage stats.
apps AS (
SELECT DISTINCT ON (ws.template_id, app.slug)
ws.template_id,
app.slug,
app.display_name,
app.icon
FROM
workspaces ws
JOIN
workspace_builds AS build
ON
build.workspace_id = ws.id
JOIN
workspace_resources AS resource
ON
resource.job_id = build.job_id
JOIN
workspace_agents AS agent
ON
agent.resource_id = resource.id
JOIN
workspace_apps AS app
ON
app.agent_id = agent.id
WHERE
-- Partial query parameter filter.
CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN ws.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
ORDER BY
ws.template_id, app.slug, app.created_at DESC
),
-- Join apps and template usage stats to filter out irrelevant rows.
-- Note that this way of joining will eliminate all data-points that
-- aren't for "real" apps. That means ports are ignored (even though
-- they're part of the dataset), as well as are "[terminal]" entries
-- which are alternate datapoints for reconnecting pty usage.
template_usage_stats_with_apps AS (
SELECT
tus.start_time,
tus.template_id,
tus.user_id,
apps.slug,
apps.display_name,
apps.icon,
(tus.app_usage_mins -> apps.slug)::smallint AS usage_mins
FROM
apps
JOIN
template_usage_stats AS tus
ON
-- Query parameter filter.
tus.start_time >= @start_time::timestamptz
AND tus.end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN tus.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
-- Primary join condition.
AND tus.template_id = apps.template_id
AND tus.app_usage_mins ? apps.slug -- Key exists in object.
),
-- Group the app insights by interval, user and unique app. This
-- allows us to deduplicate a user using the same app across
-- multiple templates.
app_insights AS (
SELECT
user_id,
slug,
display_name,
icon,
-- See motivation in GetTemplateInsights for LEAST(SUM(n), 30).
LEAST(SUM(usage_mins), 30) AS usage_mins
FROM
template_usage_stats_with_apps
GROUP BY
start_time, user_id, slug, display_name, icon
),
-- Analyze the users unique app usage across all templates. Count
-- usage across consecutive intervals as continuous usage.
times_used AS (
SELECT DISTINCT ON (user_id, slug, display_name, icon, uniq)
slug,
display_name,
icon,
-- Turn start_time into a unique identifier that identifies a users
-- continuous app usage. The value of uniq is otherwise garbage.
--
-- Since we're aggregating per user app usage across templates,
-- there can be duplicate start_times. To handle this, we use the
-- dense_rank() function, otherwise row_number() would suffice.
start_time - (
dense_rank() OVER (
PARTITION BY
user_id, slug, display_name, icon
ORDER BY
start_time
) * '30 minutes'::interval
) AS uniq
FROM
template_usage_stats_with_apps
),
-- Even though we allow identical apps to be aggregated across
-- templates, we still want to be able to report which templates
-- the data comes from.
templates AS (
SELECT
slug,
display_name,
icon,
array_agg(DISTINCT template_id)::uuid[] AS template_ids
FROM
template_usage_stats_with_apps
GROUP BY
slug, display_name, icon
)
SELECT
t.template_ids,
COUNT(DISTINCT ai.user_id) AS active_users,
ai.slug,
ai.display_name,
ai.icon,
(SUM(ai.usage_mins) * 60)::bigint AS usage_seconds,
COALESCE((
SELECT
COUNT(*)
FROM
times_used
WHERE
times_used.slug = ai.slug
AND times_used.display_name = ai.display_name
AND times_used.icon = ai.icon
), 0)::bigint AS times_used
FROM
app_insights AS ai
JOIN
templates AS t
ON
t.slug = ai.slug
AND t.display_name = ai.display_name
AND t.icon = ai.icon
GROUP BY
t.template_ids, ai.slug, ai.display_name, ai.icon;
-- name: GetTemplateAppInsightsByTemplate :many
-- GetTemplateAppInsightsByTemplate is used for Prometheus metrics. Keep
-- in sync with GetTemplateAppInsights and UpsertTemplateUsageStats.
WITH
filtered_stats AS (
SELECT
was.workspace_id,
was.user_id,
was.agent_id,
was.access_method,
was.slug_or_port,
was.session_started_at,
was.session_ended_at
FROM
workspace_app_stats AS was
WHERE
was.session_ended_at >= @start_time::timestamptz
AND was.session_started_at < @end_time::timestamptz
),
-- This CTE is used to explode app usage into minute buckets, then
-- flatten the users app usage within the template so that usage in
-- multiple workspaces under one template is only counted once for
-- every minute.
app_insights AS (
SELECT
w.template_id,
fs.user_id,
-- Both app stats and agent stats track web terminal usage, but
-- by different means. The app stats value should be more
-- accurate so we don't want to discard it just yet.
CASE
WHEN fs.access_method = 'terminal'
THEN '[terminal]' -- Unique name, app names can't contain brackets.
ELSE fs.slug_or_port
END::text AS app_name,
COALESCE(wa.display_name, '') AS display_name,
(wa.slug IS NOT NULL)::boolean AS is_app,
COUNT(DISTINCT s.minute_bucket) AS app_minutes
FROM
filtered_stats AS fs
JOIN
workspaces AS w
ON
w.id = fs.workspace_id
-- We do a left join here because we want to include user IDs that have used
-- e.g. ports when counting active users.
LEFT JOIN
workspace_apps wa
ON
wa.agent_id = fs.agent_id
AND wa.slug = fs.slug_or_port
-- Generate a series of minute buckets for each session for computing the
-- mintes/bucket.
CROSS JOIN
generate_series(
date_trunc('minute', fs.session_started_at),
-- Subtract 1 μs to avoid creating an extra series.
date_trunc('minute', fs.session_ended_at - '1 microsecond'::interval),
'1 minute'::interval
) AS s(minute_bucket)
WHERE
s.minute_bucket >= @start_time::timestamptz
AND s.minute_bucket < @end_time::timestamptz
GROUP BY
w.template_id, fs.user_id, fs.access_method, fs.slug_or_port, wa.display_name, wa.slug
)
SELECT
template_id,
app_name AS slug_or_port,
display_name AS display_name,
COUNT(DISTINCT user_id)::bigint AS active_users,
(SUM(app_minutes) * 60)::bigint AS usage_seconds
FROM
app_insights
WHERE
is_app IS TRUE
GROUP BY
template_id, slug_or_port, display_name;
-- name: GetTemplateInsightsByInterval :many
-- GetTemplateInsightsByInterval returns all intervals between start and end
-- time, if end time is a partial interval, it will be included in the results and
-- that interval will be shorter than a full one. If there is no data for a selected
-- interval/template, it will be included in the results with 0 active users.
WITH
ts AS (
SELECT
d::timestamptz AS from_,
LEAST(
(d::timestamptz + (@interval_days::int || ' day')::interval)::timestamptz,
@end_time::timestamptz
)::timestamptz AS to_
FROM
generate_series(
@start_time::timestamptz,
-- Subtract 1 μs to avoid creating an extra series.
(@end_time::timestamptz) - '1 microsecond'::interval,
(@interval_days::int || ' day')::interval
) AS d
)
SELECT
ts.from_ AS start_time,
ts.to_ AS end_time,
array_remove(array_agg(DISTINCT tus.template_id), NULL)::uuid[] AS template_ids,
COUNT(DISTINCT tus.user_id) AS active_users
FROM
ts
LEFT JOIN
template_usage_stats AS tus
ON
tus.start_time >= ts.from_
AND tus.start_time < ts.to_ -- End time exclusion criteria optimization for index.
AND tus.end_time <= ts.to_
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN tus.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
ts.from_, ts.to_;
-- name: GetTemplateUsageStats :many
SELECT
*
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END;
-- name: UpsertTemplateUsageStats :exec
-- This query aggregates the workspace_agent_stats and workspace_app_stats data
-- into a single table for efficient storage and querying. Half-hour buckets are
-- used to store the data, and the minutes are summed for each user and template
-- combination. The result is stored in the template_usage_stats table.
--
-- Session usage is stored per app name in the child table, so the main row
-- carries no session columns at all. Every recomputed bucket rewrites its own
-- child rows: app names that disappeared are deleted, the rest are upserted.
-- The keys come from the computed set rather than from the main upsert,
-- because the no-op guard below suppresses main rows whose columns did not
-- change while their session usage still has to be corrected.
WITH
latest_start AS (
SELECT
-- Truncate to hour so that we always look at even ranges of data.
date_trunc('hour', COALESCE(
MAX(start_time) - '1 hour'::interval,
-- Fallback when there are no template usage stats yet.
-- App stats can exist before this, but not agent stats,
-- limit the lookback to avoid inconsistency.
(SELECT MIN(created_at) FROM workspace_agent_stats)
)) AS t
FROM
template_usage_stats
),
filtered_app_stats AS (
SELECT
was.workspace_id,
was.user_id,
was.agent_id,
was.access_method,
was.slug_or_port,
was.session_started_at,
was.session_ended_at
FROM
workspace_app_stats AS was
WHERE
was.session_ended_at >= (SELECT t FROM latest_start)
AND was.session_started_at < NOW()
),
workspace_app_stat_buckets AS (
SELECT
-- Truncate the minute to the nearest half hour, this is the bucket size
-- for the data.
date_trunc('hour', s.minute_bucket) + trunc(date_part('minute', s.minute_bucket) / 30) * 30 * '1 minute'::interval AS time_bucket,
w.template_id,
fas.user_id,
-- Both app stats and agent stats track web terminal usage, but
-- by different means. The app stats value should be more
-- accurate so we don't want to discard it just yet.
CASE
WHEN fas.access_method = 'terminal'
THEN '[terminal]' -- Unique name, app names can't contain brackets.
ELSE fas.slug_or_port
END AS app_name,
COUNT(DISTINCT s.minute_bucket) AS app_minutes,
-- Store each unique minute bucket for later merge between datasets.
array_agg(DISTINCT s.minute_bucket) AS minute_buckets
FROM
filtered_app_stats AS fas
JOIN
workspaces AS w
ON
w.id = fas.workspace_id
-- Generate a series of minute buckets for each session for computing the
-- mintes/bucket.
CROSS JOIN
generate_series(
date_trunc('minute', fas.session_started_at),
-- Subtract 1 μs to avoid creating an extra series.
date_trunc('minute', fas.session_ended_at - '1 microsecond'::interval),
'1 minute'::interval
) AS s(minute_bucket)
WHERE
-- s.minute_bucket >= @start_time::timestamptz
-- AND s.minute_bucket < @end_time::timestamptz
s.minute_bucket >= (SELECT t FROM latest_start)
AND s.minute_bucket < NOW()
GROUP BY
time_bucket, w.template_id, fas.user_id, fas.access_method, fas.slug_or_port
),
agent_stats_rows AS (
-- One filtered pass feeds both groupings below.
SELECT
date_trunc('hour', created_at) + trunc(date_part('minute', created_at) / 30) * 30 * '1 minute'::interval AS time_bucket,
template_id,
user_id,
date_trunc('minute', created_at) AS minute_bucket,
connection_count,
session_counts
FROM
workspace_agent_stats
WHERE
-- created_at >= @start_time::timestamptz
-- AND created_at < @end_time::timestamptz
created_at >= (SELECT t FROM latest_start)
AND created_at < NOW()
AND session_counts <> '{}'::jsonb
),
agent_stats_buckets AS (
SELECT
time_bucket,
template_id,
user_id,
-- Store each unique minute bucket for later merge between datasets.
array_agg(DISTINCT minute_bucket) AS minute_buckets,
-- NOTE(mafredri): connection_count covers one report interval,
-- while the session counts are a gauge, so an idle session reports
-- none. Hence "any connection during this half-hour", pending a
-- better solution.
MAX(connection_count) > 0 AS has_connection
FROM
agent_stats_rows
GROUP BY
time_bucket, template_id, user_id
),
agent_stats_session_minutes AS (
-- A minute counts once per app however many of its sessions were open.
SELECT
agent_stats.time_bucket,
agent_stats.template_id,
agent_stats.user_id,
app_name,
COUNT(DISTINCT agent_stats.minute_bucket)::smallint AS usage_mins
FROM
agent_stats_rows AS agent_stats
JOIN
agent_stats_buckets AS buckets
ON
buckets.time_bucket = agent_stats.time_bucket
AND buckets.template_id = agent_stats.template_id
AND buckets.user_id = agent_stats.user_id
-- Same gate as the union below, so an app-stats-only bucket
-- records no session usage.
AND buckets.has_connection
CROSS JOIN
jsonb_object_keys(agent_stats.session_counts) AS app_name
GROUP BY
agent_stats.time_bucket, agent_stats.template_id, agent_stats.user_id, app_name
),
stats AS (
SELECT
stats.time_bucket AS start_time,
stats.time_bucket + '30 minutes'::interval AS end_time,
stats.template_id,
stats.user_id,
-- Distinct to handle duplicate values due union and to unnest.
COUNT(DISTINCT minute_bucket) AS usage_mins,
array_agg(DISTINCT minute_bucket) AS minute_buckets,
-- This is what we unnested, re-nest as json.
jsonb_object_agg(stats.app_name, stats.app_minutes) FILTER (WHERE stats.app_name IS NOT NULL) AS app_usage_mins
FROM (
SELECT
time_bucket,
template_id,
user_id,
app_name,
app_minutes,
minute_buckets
FROM
workspace_app_stat_buckets
UNION ALL
SELECT
time_bucket,
template_id,
user_id,
NULL AS app_name,
NULL AS app_minutes,
minute_buckets
FROM
agent_stats_buckets
WHERE
-- See note in the agent_stats_buckets CTE.
has_connection
) AS stats, unnest(minute_buckets) AS minute_bucket
GROUP BY
stats.time_bucket, stats.template_id, stats.user_id
),
minute_buckets AS (
-- Create distinct minute buckets for user-activity, so we can filter out
-- irrelevant latencies.
SELECT DISTINCT ON (stats.start_time, stats.template_id, stats.user_id, minute_bucket)
stats.start_time,
stats.template_id,
stats.user_id,
minute_bucket
FROM
stats, unnest(minute_buckets) AS minute_bucket
),
latencies AS (
-- Select all non-zero latencies for all the minutes that a user used the
-- workspace in some way.
SELECT
mb.start_time,
mb.template_id,
mb.user_id,
-- TODO(mafredri): We're doing medians on medians here, we may want to
-- improve upon this at some point.
PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY was.connection_median_latency_ms)::real AS median_latency_ms
FROM
minute_buckets AS mb
JOIN
workspace_agent_stats AS was
ON
was.created_at >= (SELECT t FROM latest_start)
AND was.created_at < NOW()
AND date_trunc('minute', was.created_at) = mb.minute_bucket
AND was.template_id = mb.template_id
AND was.user_id = mb.user_id
AND was.connection_median_latency_ms > 0
GROUP BY
mb.start_time, mb.template_id, mb.user_id
),
upsert_stats AS (
INSERT INTO template_usage_stats AS tus (
start_time,
end_time,
template_id,
user_id,
usage_mins,
median_latency_ms,
app_usage_mins
) (
SELECT
stats.start_time,
stats.end_time,
stats.template_id,
stats.user_id,
stats.usage_mins,
latencies.median_latency_ms,
stats.app_usage_mins
FROM
stats
LEFT JOIN
latencies
ON
-- The latencies group-by ensures there at most one row.
latencies.start_time = stats.start_time
AND latencies.template_id = stats.template_id
AND latencies.user_id = stats.user_id
)
ON CONFLICT
(start_time, template_id, user_id)
DO UPDATE
SET
usage_mins = EXCLUDED.usage_mins,
median_latency_ms = EXCLUDED.median_latency_ms,
app_usage_mins = EXCLUDED.app_usage_mins
WHERE
(tus.*) IS DISTINCT FROM (EXCLUDED.*)
),
-- The child writes share this statement, so the foreign keys are checked
-- once it completes and see the main rows the upsert added; Postgres runs
-- that upsert whether or not the statement reads it.
--
-- NOT IN, not NOT EXISTS: the planner has no statistics for the CTE and
-- turns NOT EXISTS into a nested loop that rescans it per row, 20 seconds
-- per rollup. Every column is non-null, so both delete the same rows.
delete_apps AS (
DELETE FROM
template_usage_stats_session_apps AS apps
USING
agent_stats_buckets AS buckets
WHERE
apps.start_time = buckets.time_bucket
AND apps.template_id = buckets.template_id
AND apps.user_id = buckets.user_id
AND (apps.start_time, apps.template_id, apps.user_id, apps.app_name) NOT IN (
SELECT time_bucket, template_id, user_id, app_name
FROM agent_stats_session_minutes
)
)
INSERT INTO template_usage_stats_session_apps AS apps (
start_time,
template_id,
user_id,
app_name,
usage_mins
) (
SELECT
time_bucket,
template_id,
user_id,
app_name,
usage_mins
FROM
agent_stats_session_minutes
)
ON CONFLICT
(start_time, template_id, user_id, app_name)
DO UPDATE
SET
usage_mins = EXCLUDED.usage_mins
WHERE
apps.usage_mins IS DISTINCT FROM EXCLUDED.usage_mins;
-- name: GetTemplateParameterInsights :many
-- GetTemplateParameterInsights does for each template in a given timeframe,
-- look for the latest workspace build (for every workspace) that has been
-- created in the timeframe and return the aggregate usage counts of parameter
-- values.
WITH latest_workspace_builds AS (
SELECT
wb.id,
wbmax.template_id,
wb.template_version_id
FROM (
SELECT
tv.template_id, wbmax.workspace_id, MAX(wbmax.build_number) as max_build_number
FROM workspace_builds wbmax
JOIN template_versions tv ON (tv.id = wbmax.template_version_id)
WHERE
wbmax.created_at >= @start_time::timestamptz
AND wbmax.created_at < @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN tv.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY tv.template_id, wbmax.workspace_id
) wbmax
JOIN workspace_builds wb ON (
wb.workspace_id = wbmax.workspace_id
AND wb.build_number = wbmax.max_build_number
)
), unique_template_params AS (
SELECT
ROW_NUMBER() OVER () AS num,
array_agg(DISTINCT wb.template_id)::uuid[] AS template_ids,
array_agg(wb.id)::uuid[] AS workspace_build_ids,
tvp.name,
tvp.type,
tvp.display_name,
tvp.description,
tvp.options
FROM latest_workspace_builds wb
JOIN template_version_parameters tvp ON (tvp.template_version_id = wb.template_version_id)
GROUP BY tvp.name, tvp.type, tvp.display_name, tvp.description, tvp.options
)
SELECT
utp.num,
utp.template_ids,
utp.name,
utp.type,
utp.display_name,
utp.description,
utp.options,
wbp.value,
COUNT(wbp.value) AS count
FROM unique_template_params utp
JOIN workspace_build_parameters wbp ON (utp.workspace_build_ids @> ARRAY[wbp.workspace_build_id] AND utp.name = wbp.name)
GROUP BY utp.num, utp.template_ids, utp.name, utp.type, utp.display_name, utp.description, utp.options, wbp.value;
-- name: GetUserStatusCounts :many
-- GetUserStatusCounts returns the count of users in each status over time.
-- The time range is inclusively defined by the start_time and end_time parameters.
WITH
system_users AS (
SELECT id FROM users WHERE is_system = TRUE
),
-- dates_of_interest generates the dates that will represent the horizontal axis of the chart.
dates_of_interest AS (
SELECT timezone(@tz::text, gs_local) AS date
FROM generate_series(
timezone(@tz::text, @start_time::timestamptz),
timezone(@tz::text, @end_time::timestamptz),
interval '1 day'
) AS gs_local
),
-- latest_status_before_range selects the last status of each user before the start_time.
-- This represents the status of all users at the start of the time range.
latest_status_before_range AS (
SELECT
DISTINCT usc.user_id,
usc.new_status,
usc.changed_at
FROM user_status_changes usc
LEFT JOIN LATERAL (
SELECT COUNT(*) > 0 AS deleted
FROM user_deleted ud
WHERE ud.user_id = usc.user_id AND (ud.deleted_at < usc.changed_at OR ud.deleted_at < @start_time)
) AS ud ON true
WHERE usc.user_id NOT IN (SELECT id FROM system_users)
AND NOT ud.deleted
AND usc.changed_at < @start_time::timestamptz
ORDER BY usc.user_id, usc.changed_at DESC
),
-- status_changes_during_range selects the statuses of each user during the start_time and end_time.
status_changes_during_range AS (
SELECT
usc.user_id,
usc.new_status,
usc.changed_at
FROM user_status_changes usc
LEFT JOIN LATERAL (
SELECT COUNT(*) > 0 AS deleted
FROM user_deleted ud
WHERE ud.user_id = usc.user_id AND ud.deleted_at < usc.changed_at
) AS ud ON true
WHERE usc.user_id NOT IN (SELECT id FROM system_users)
AND NOT ud.deleted
AND usc.changed_at >= @start_time::timestamptz
AND usc.changed_at <= @end_time::timestamptz
),
relevant_status_changes AS (
SELECT user_id, new_status, changed_at
FROM latest_status_before_range
UNION ALL
SELECT user_id, new_status, changed_at
FROM status_changes_during_range
),
-- statuses selects all the distinct statuses that were present just before and during the time range.
-- Each status will have a series on the chart.
statuses AS (
SELECT DISTINCT new_status FROM relevant_status_changes
),
-- ranked_status_change_per_user_per_date selects the latest status change for each user on each date.
-- The last status for a user on every given date will be counted.
ranked_status_change_per_user_per_date AS (
SELECT
d.date,
rsc1.user_id,
ROW_NUMBER() OVER (PARTITION BY d.date, rsc1.user_id ORDER BY rsc1.changed_at DESC) AS rn,
rsc1.new_status
FROM dates_of_interest d
LEFT JOIN relevant_status_changes rsc1 ON rsc1.changed_at <= d.date
)
SELECT
rscpupd.date::timestamptz AS date,
statuses.new_status AS status,
COUNT(rscpupd.user_id) FILTER (
WHERE rscpupd.rn = 1
AND (
rscpupd.new_status = statuses.new_status
AND (
-- Include users who haven't been deleted
NOT EXISTS (SELECT 1 FROM user_deleted WHERE user_id = rscpupd.user_id)
OR
-- Or users whose deletion date is after the current date we're looking at
rscpupd.date < (SELECT deleted_at FROM user_deleted WHERE user_id = rscpupd.user_id)
)
)
) AS count
FROM ranked_status_change_per_user_per_date rscpupd
CROSS JOIN statuses
GROUP BY rscpupd.date, statuses.new_status
ORDER BY rscpupd.date;