import { msg } from "@lingui/core/macro"; import { pbAdmin } from "@/lib/pocketbase/client"; import { sendPushToPlayer } from "@/lib/push"; import { Logger } from "@/lib/logger"; import { ADMIN_ROLE } from "@/features/core/utils/roles"; const logger = new Logger("Telemetry > Alerts"); const WINDOW_MS = 15 * 60 * 1000; const COOLDOWN_MS = 60 * 60 * 1000; const MAX_NEW_GROUPS_PER_RUN = 20; const errorSpikeThreshold = (): number => { const raw = Number(process.env.TELEMETRY_ALERT_ERROR_COUNT); return Number.isFinite(raw) && raw > 0 ? raw : 10; }; const fnFailureThreshold = (): number => { const raw = Number(process.env.TELEMETRY_ALERT_FN_FAILURES); return Number.isFinite(raw) && raw > 0 ? raw : 5; }; const pbTimestamp = (date: Date): string => date.toISOString().replace("T", " "); const ADMIN_CACHE_TTL_MS = 10 * 60 * 1000; let adminPlayersCache: { playerIds: string[]; expiresAt: number } | null = null; const getAdminPlayerIds = async (): Promise => { const now = Date.now(); if (adminPlayersCache && adminPlayersCache.expiresAt > now) { return adminPlayersCache.playerIds; } const { ensureSuperTokensBackend } = await import("@/lib/supertokens/server"); ensureSuperTokensBackend(); const UserRoles = (await import("supertokens-node/recipe/userroles")).default; const UserMetadata = (await import("supertokens-node/recipe/usermetadata")).default; const response = await UserRoles.getUsersThatHaveRole("public", ADMIN_ROLE); const users = response.status === "OK" ? response.users : []; const playerIds: string[] = []; for (const userId of users) { try { // Opt-in: only admins who enabled telemetry alerts in settings. const { metadata } = await UserMetadata.getUserMetadata(userId); if (metadata?.telemetryAlerts !== true) continue; const player = await pbAdmin.getPlayerByAuthId(userId); if (player) playerIds.push(player.id); } catch {} } adminPlayersCache = { playerIds, expiresAt: now + ADMIN_CACHE_TTL_MS }; return playerIds; }; interface AlertCandidate { kind: string; dim: string; message: string; value: number; title: Parameters[1]["title"]; body: Parameters[1]["body"]; } const fireAlert = async (candidate: AlertCandidate) => { const since = new Date(Date.now() - COOLDOWN_MS).toISOString(); const recent = await pbAdmin.findRecentAlert(candidate.kind, candidate.dim, pbTimestamp(new Date(since))); if (recent) return; await pbAdmin.createTelemetryAlert({ kind: candidate.kind, dim: candidate.dim, message: candidate.message, value: candidate.value, }); const admins = await getAdminPlayerIds(); for (const playerId of admins) { try { await sendPushToPlayer(playerId, { title: candidate.title, body: candidate.body, url: "/admin/telemetry/errors", tag: `telemetry-${candidate.kind}`, }); } catch (error) { logger.error("Failed to push telemetry alert", error); } } logger.info(`Alert fired: ${candidate.kind} (${candidate.dim || "-"}) = ${candidate.value}`); }; export const runAlertChecks = async (): Promise => { await pbAdmin.authPromise; const windowStart = pbTimestamp(new Date(Date.now() - WINDOW_MS)); const recentErrors = await pbAdmin.searchClientErrors({ from: windowStart, perPage: 200, sortBy: "-created", }); if (recentErrors.totalItems >= errorSpikeThreshold()) { await fireAlert({ kind: "client_error_spike", dim: "", message: `${recentErrors.totalItems} client errors in 15 minutes`, value: recentErrors.totalItems, title: msg`Client error spike`, body: msg`${recentErrors.totalItems} client errors in the last 15 minutes`, }); } const groups = new Map(); for (const error of recentErrors.items) { const entry = groups.get(error.group_hash) ?? { count: 0, message: error.message }; entry.count++; groups.set(error.group_hash, entry); } let checked = 0; for (const [groupHash, info] of groups) { if (checked++ >= MAX_NEW_GROUPS_PER_RUN) break; const older = await pbAdmin.searchClientErrors({ groupHash, to: windowStart, perPage: 1 }); if (older.totalItems === 0) { await fireAlert({ kind: "client_error_new_group", dim: groupHash, message: info.message, value: info.count, title: msg`New client error`, body: msg`A new error appeared: ${info.message.slice(0, 120)}`, }); } } const failedActivities = await pbAdmin.searchActivities({ success: false, perPage: 200, sortBy: "-created", }); const failuresByFn = new Map(); for (const activity of failedActivities.items) { if (activity.created < windowStart) continue; if (activity.error?.startsWith("FORBIDDEN")) { failuresByFn.set("__denied__", (failuresByFn.get("__denied__") ?? 0) + 1); continue; } failuresByFn.set(activity.name, (failuresByFn.get(activity.name) ?? 0) + 1); } for (const [name, count] of failuresByFn) { if (name === "__denied__") continue; if (count >= fnFailureThreshold()) { await fireAlert({ kind: "server_fn_failures", dim: name, message: `${name} failed ${count} times in 15 minutes`, value: count, title: msg`Server function failing`, body: msg`${name} failed ${count} times in the last 15 minutes`, }); } } const deniedCount = failuresByFn.get("__denied__") ?? 0; if (deniedCount >= 1) { await fireAlert({ kind: "denied_access", dim: "", message: `${deniedCount} denied admin access attempts in 15 minutes`, value: deniedCount, title: msg`Denied access attempt`, body: msg`${deniedCount} denied admin access attempts in the last 15 minutes`, }); } };