feat(telemetry): rollups, alerts, scheduler, health checks, admin server fns

- pure rollup accumulators with tests; daily upserts keyed date/metric/dim
- alert checks with db-backed cooldown and push to admin players
- in-process scheduler started from health probes
- health endpoint checks pb + supertokens, stale-while-revalidate, always 200
- sse connection counter moved to emitter
- telemetry feature server fns: events/errors/groups/rollups/alerts/status

src/lib/telemetry/rollup-core.ts

@@ -0,0 +1,175 @@
+import type { RollupInput } from "@/lib/pocketbase/services/telemetry";
+
+export interface ActivityRow {
+  name: string;
+  player?: string;
+  duration?: number;
+  success?: boolean;
+  error?: string;
+}
+
+export interface ClientEventRow {
+  name: string;
+  player?: string;
+  session_id?: string;
+  route_id?: string;
+  value?: number;
+}
+
+export interface ClientErrorRow {
+  group_hash: string;
+}
+
+// Nearest-rank percentile; values need not be pre-sorted.
+export const percentile = (values: number[], p: number): number => {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const rank = Math.ceil((p / 100) * sorted.length);
+  return sorted[Math.min(sorted.length, Math.max(1, rank)) - 1];
+};
+
+export interface ActivityAccumulator {
+  players: Set<string>;
+  perFn: Map<string, { count: number; errors: number; durations: number[] }>;
+  denied: number;
+  otpSent: number;
+  otpFailed: number;
+}
+
+export const newActivityAccumulator = (): ActivityAccumulator => ({
+  players: new Set(),
+  perFn: new Map(),
+  denied: 0,
+  otpSent: 0,
+  otpFailed: 0,
+});
+
+export const accumulateActivities = (acc: ActivityAccumulator, rows: ActivityRow[]) => {
+  for (const row of rows) {
+    if (row.player) acc.players.add(row.player);
+    if (row.error?.startsWith("FORBIDDEN")) acc.denied++;
+    if (row.name === "auth.otp_sent") acc.otpSent++;
+    if (row.name === "auth.otp_failed") acc.otpFailed++;
+    if (row.name.startsWith("auth.")) continue;
+
+    const fn = acc.perFn.get(row.name) ?? { count: 0, errors: 0, durations: [] };
+    fn.count++;
+    if (row.success === false) fn.errors++;
+    if (typeof row.duration === "number" && row.duration > 0) fn.durations.push(row.duration);
+    acc.perFn.set(row.name, fn);
+  }
+};
+
+export interface ClientEventAccumulator {
+  players: Set<string>;
+  sessions: Set<string>;
+  pageViewsByRoute: Map<string, number>;
+  vitals: Map<string, number[]>;
+}
+
+export const newClientEventAccumulator = (): ClientEventAccumulator => ({
+  players: new Set(),
+  sessions: new Set(),
+  pageViewsByRoute: new Map(),
+  vitals: new Map(),
+});
+
+export const accumulateClientEvents = (acc: ClientEventAccumulator, rows: ClientEventRow[]) => {
+  for (const row of rows) {
+    if (row.player) acc.players.add(row.player);
+    if (row.session_id) acc.sessions.add(row.session_id);
+
+    if (row.name === "page_view") {
+      const route = row.route_id || "(unknown)";
+      acc.pageViewsByRoute.set(route, (acc.pageViewsByRoute.get(route) ?? 0) + 1);
+    } else if (row.name.startsWith("vital.") && typeof row.value === "number") {
+      const metric = row.name.slice("vital.".length);
+      const values = acc.vitals.get(metric) ?? [];
+      values.push(row.value);
+      acc.vitals.set(metric, values);
+    }
+  }
+};
+
+export interface ClientErrorAccumulator {
+  byGroup: Map<string, number>;
+  total: number;
+}
+
+export const newClientErrorAccumulator = (): ClientErrorAccumulator => ({
+  byGroup: new Map(),
+  total: 0,
+});
+
+export const accumulateClientErrors = (acc: ClientErrorAccumulator, rows: ClientErrorRow[]) => {
+  for (const row of rows) {
+    acc.total++;
+    acc.byGroup.set(row.group_hash, (acc.byGroup.get(row.group_hash) ?? 0) + 1);
+  }
+};
+
+export const buildDayRollups = (
+  date: string,
+  input: {
+    activities: ActivityAccumulator;
+    clientEvents: ClientEventAccumulator;
+    clientErrors: ClientErrorAccumulator;
+    wau: number;
+    mau: number;
+    partial: boolean;
+  }
+): RollupInput[] => {
+  const { activities, clientEvents, clientErrors, wau, mau, partial } = input;
+  const meta = partial ? { partial: true } : undefined;
+  const rollups: RollupInput[] = [];
+  const push = (metric: string, dim: string, value: number, extraMeta?: Record<string, unknown>) => {
+    rollups.push({
+      date,
+      metric,
+      dim,
+      value,
+      meta: extraMeta ? { ...meta, ...extraMeta } : meta,
+    });
+  };
+
+  const dayPlayers = new Set([...activities.players, ...clientEvents.players]);
+  push("dau", "", dayPlayers.size);
+  push("wau", "", wau);
+  push("mau", "", mau);
+
+  for (const [name, fn] of activities.perFn) {
+    push("server_fn.count", name, fn.count);
+    if (fn.errors > 0) push("server_fn.error_count", name, fn.errors);
+    if (fn.durations.length > 0) {
+      push("server_fn.duration.p50", name, percentile(fn.durations, 50), {
+        samples: fn.durations.length,
+      });
+      push("server_fn.duration.p95", name, percentile(fn.durations, 95), {
+        samples: fn.durations.length,
+      });
+    }
+  }
+
+  let totalPageViews = 0;
+  for (const [route, count] of clientEvents.pageViewsByRoute) {
+    push("page_view.count", route, count);
+    totalPageViews += count;
+  }
+  push("page_view.count", "", totalPageViews);
+  push("page_view.sessions", "", clientEvents.sessions.size);
+
+  push("client_error.count", "", clientErrors.total);
+  for (const [group, count] of clientErrors.byGroup) {
+    push("client_error.count", group, count);
+  }
+
+  for (const [metric, values] of clientEvents.vitals) {
+    push("vital.p75", metric, percentile(values, 75), { samples: values.length });
+  }
+
+  push("denied.count", "", activities.denied);
+  push("auth.otp_sent.count", "", activities.otpSent);
+  push("auth.otp_failed.count", "", activities.otpFailed);
+
+  return rollups;
+};