feat(telemetry): rollups, alerts, scheduler, health checks, admin server fns

- pure rollup accumulators with tests; daily upserts keyed date/metric/dim
- alert checks with db-backed cooldown and push to admin players
- in-process scheduler started from health probes
- health endpoint checks pb + supertokens, stale-while-revalidate, always 200
- sse connection counter moved to emitter
- telemetry feature server fns: events/errors/groups/rollups/alerts/status

src/lib/telemetry/alerts.server.ts

@@ -0,0 +1,170 @@
+import { msg } from "@lingui/core/macro";
+import { pbAdmin } from "@/lib/pocketbase/client";
+import { sendPushToPlayer } from "@/lib/push";
+import { Logger } from "@/lib/logger";
+import { ADMIN_ROLE } from "@/features/core/utils/roles";
+
+const logger = new Logger("Telemetry > Alerts");
+
+const WINDOW_MS = 15 * 60 * 1000;
+const COOLDOWN_MS = 60 * 60 * 1000;
+const MAX_NEW_GROUPS_PER_RUN = 20;
+
+const errorSpikeThreshold = (): number => {
+  const raw = Number(process.env.TELEMETRY_ALERT_ERROR_COUNT);
+  return Number.isFinite(raw) && raw > 0 ? raw : 10;
+};
+
+const fnFailureThreshold = (): number => {
+  const raw = Number(process.env.TELEMETRY_ALERT_FN_FAILURES);
+  return Number.isFinite(raw) && raw > 0 ? raw : 5;
+};
+
+const pbTimestamp = (date: Date): string => date.toISOString().replace("T", " ");
+
+const ADMIN_CACHE_TTL_MS = 10 * 60 * 1000;
+let adminPlayersCache: { playerIds: string[]; expiresAt: number } | null = null;
+
+const getAdminPlayerIds = async (): Promise<string[]> => {
+  const now = Date.now();
+  if (adminPlayersCache && adminPlayersCache.expiresAt > now) {
+    return adminPlayersCache.playerIds;
+  }
+
+  const UserRoles = (await import("supertokens-node/recipe/userroles")).default;
+  const response = await UserRoles.getUsersThatHaveRole("public", ADMIN_ROLE);
+  const users = response.status === "OK" ? response.users : [];
+  const playerIds: string[] = [];
+  for (const userId of users) {
+    try {
+      const player = await pbAdmin.getPlayerByAuthId(userId);
+      if (player) playerIds.push(player.id);
+    } catch {}
+  }
+
+  adminPlayersCache = { playerIds, expiresAt: now + ADMIN_CACHE_TTL_MS };
+  return playerIds;
+};
+
+interface AlertCandidate {
+  kind: string;
+  dim: string;
+  message: string;
+  value: number;
+  title: Parameters<typeof sendPushToPlayer>[1]["title"];
+  body: Parameters<typeof sendPushToPlayer>[1]["body"];
+}
+
+const fireAlert = async (candidate: AlertCandidate) => {
+  const since = new Date(Date.now() - COOLDOWN_MS).toISOString();
+  const recent = await pbAdmin.findRecentAlert(candidate.kind, candidate.dim, pbTimestamp(new Date(since)));
+  if (recent) return;
+
+  await pbAdmin.createTelemetryAlert({
+    kind: candidate.kind,
+    dim: candidate.dim,
+    message: candidate.message,
+    value: candidate.value,
+  });
+
+  const admins = await getAdminPlayerIds();
+  for (const playerId of admins) {
+    try {
+      await sendPushToPlayer(playerId, {
+        title: candidate.title,
+        body: candidate.body,
+        url: "/admin/telemetry/errors",
+        tag: `telemetry-${candidate.kind}`,
+      });
+    } catch (error) {
+      logger.error("Failed to push telemetry alert", error);
+    }
+  }
+
+  logger.info(`Alert fired: ${candidate.kind} (${candidate.dim || "-"}) = ${candidate.value}`);
+};
+
+export const runAlertChecks = async (): Promise<void> => {
+  await pbAdmin.authPromise;
+  const windowStart = pbTimestamp(new Date(Date.now() - WINDOW_MS));
+
+  const recentErrors = await pbAdmin.searchClientErrors({
+    from: windowStart,
+    perPage: 200,
+    sortBy: "-created",
+  });
+
+  if (recentErrors.totalItems >= errorSpikeThreshold()) {
+    await fireAlert({
+      kind: "client_error_spike",
+      dim: "",
+      message: `${recentErrors.totalItems} client errors in 15 minutes`,
+      value: recentErrors.totalItems,
+      title: msg`Client error spike`,
+      body: msg`${recentErrors.totalItems} client errors in the last 15 minutes`,
+    });
+  }
+
+  const groups = new Map<string, { count: number; message: string }>();
+  for (const error of recentErrors.items) {
+    const entry = groups.get(error.group_hash) ?? { count: 0, message: error.message };
+    entry.count++;
+    groups.set(error.group_hash, entry);
+  }
+  let checked = 0;
+  for (const [groupHash, info] of groups) {
+    if (checked++ >= MAX_NEW_GROUPS_PER_RUN) break;
+    const older = await pbAdmin.searchClientErrors({ groupHash, to: windowStart, perPage: 1 });
+    if (older.totalItems === 0) {
+      await fireAlert({
+        kind: "client_error_new_group",
+        dim: groupHash,
+        message: info.message,
+        value: info.count,
+        title: msg`New client error`,
+        body: msg`A new error appeared: ${info.message.slice(0, 120)}`,
+      });
+    }
+  }
+
+  const failedActivities = await pbAdmin.searchActivities({
+    success: false,
+    perPage: 200,
+    sortBy: "-created",
+  });
+  const failuresByFn = new Map<string, number>();
+  for (const activity of failedActivities.items) {
+    if (activity.created < windowStart) continue;
+    if (activity.error?.startsWith("FORBIDDEN")) {
+      failuresByFn.set("__denied__", (failuresByFn.get("__denied__") ?? 0) + 1);
+      continue;
+    }
+    failuresByFn.set(activity.name, (failuresByFn.get(activity.name) ?? 0) + 1);
+  }
+
+  for (const [name, count] of failuresByFn) {
+    if (name === "__denied__") continue;
+    if (count >= fnFailureThreshold()) {
+      await fireAlert({
+        kind: "server_fn_failures",
+        dim: name,
+        message: `${name} failed ${count} times in 15 minutes`,
+        value: count,
+        title: msg`Server function failing`,
+        body: msg`${name} failed ${count} times in the last 15 minutes`,
+      });
+    }
+  }
+
+  const deniedCount = failuresByFn.get("__denied__") ?? 0;
+  if (deniedCount >= 1) {
+    await fireAlert({
+      kind: "denied_access",
+      dim: "",
+      message: `${deniedCount} denied admin access attempts in 15 minutes`,
+      value: deniedCount,
+      title: msg`Denied access attempt`,
+      body: msg`${deniedCount} denied admin access attempts in the last 15 minutes`,
+    });
+  }
+};