flxn-app/src/lib/telemetry/alerts.server.ts

import { msg } from "@lingui/core/macro";
import { pbAdmin } from "@/lib/pocketbase/client";
import { sendPushToPlayer } from "@/lib/push";
import { Logger } from "@/lib/logger";
import { ADMIN_ROLE } from "@/features/core/utils/roles";

const logger = new Logger("Telemetry > Alerts");

const WINDOW_MS = 15 * 60 * 1000;
const COOLDOWN_MS = 60 * 60 * 1000;
const MAX_NEW_GROUPS_PER_RUN = 20;

const errorSpikeThreshold = (): number => {
  const raw = Number(process.env.TELEMETRY_ALERT_ERROR_COUNT);
  return Number.isFinite(raw) && raw > 0 ? raw : 10;
};

const fnFailureThreshold = (): number => {
  const raw = Number(process.env.TELEMETRY_ALERT_FN_FAILURES);
  return Number.isFinite(raw) && raw > 0 ? raw : 5;
};

const pbTimestamp = (date: Date): string => date.toISOString().replace("T", " ");

const ADMIN_CACHE_TTL_MS = 10 * 60 * 1000;
let adminPlayersCache: { playerIds: string[]; expiresAt: number } | null = null;

const getAdminPlayerIds = async (): Promise<string[]> => {
  const now = Date.now();
  if (adminPlayersCache && adminPlayersCache.expiresAt > now) {
    return adminPlayersCache.playerIds;
  }

  const UserRoles = (await import("supertokens-node/recipe/userroles")).default;
  const UserMetadata = (await import("supertokens-node/recipe/usermetadata")).default;
  const response = await UserRoles.getUsersThatHaveRole("public", ADMIN_ROLE);
  const users = response.status === "OK" ? response.users : [];
  const playerIds: string[] = [];
  for (const userId of users) {
    try {
      // Opt-in: only admins who enabled telemetry alerts in settings.
      const { metadata } = await UserMetadata.getUserMetadata(userId);
      if (metadata?.telemetryAlerts !== true) continue;
      const player = await pbAdmin.getPlayerByAuthId(userId);
      if (player) playerIds.push(player.id);
    } catch {}
  }

  adminPlayersCache = { playerIds, expiresAt: now + ADMIN_CACHE_TTL_MS };
  return playerIds;
};

interface AlertCandidate {
  kind: string;
  dim: string;
  message: string;
  value: number;
  title: Parameters<typeof sendPushToPlayer>[1]["title"];
  body: Parameters<typeof sendPushToPlayer>[1]["body"];
}

const fireAlert = async (candidate: AlertCandidate) => {
  const since = new Date(Date.now() - COOLDOWN_MS).toISOString();
  const recent = await pbAdmin.findRecentAlert(candidate.kind, candidate.dim, pbTimestamp(new Date(since)));
  if (recent) return;

  await pbAdmin.createTelemetryAlert({
    kind: candidate.kind,
    dim: candidate.dim,
    message: candidate.message,
    value: candidate.value,
  });

  const admins = await getAdminPlayerIds();
  for (const playerId of admins) {
    try {
      await sendPushToPlayer(playerId, {
        title: candidate.title,
        body: candidate.body,
        url: "/admin/telemetry/errors",
        tag: `telemetry-${candidate.kind}`,
      });
    } catch (error) {
      logger.error("Failed to push telemetry alert", error);
    }
  }

  logger.info(`Alert fired: ${candidate.kind} (${candidate.dim || "-"}) = ${candidate.value}`);
};

export const runAlertChecks = async (): Promise<void> => {
  await pbAdmin.authPromise;
  const windowStart = pbTimestamp(new Date(Date.now() - WINDOW_MS));

  const recentErrors = await pbAdmin.searchClientErrors({
    from: windowStart,
    perPage: 200,
    sortBy: "-created",
  });

  if (recentErrors.totalItems >= errorSpikeThreshold()) {
    await fireAlert({
      kind: "client_error_spike",
      dim: "",
      message: `${recentErrors.totalItems} client errors in 15 minutes`,
      value: recentErrors.totalItems,
      title: msg`Client error spike`,
      body: msg`${recentErrors.totalItems} client errors in the last 15 minutes`,
    });
  }

  const groups = new Map<string, { count: number; message: string }>();
  for (const error of recentErrors.items) {
    const entry = groups.get(error.group_hash) ?? { count: 0, message: error.message };
    entry.count++;
    groups.set(error.group_hash, entry);
  }
  let checked = 0;
  for (const [groupHash, info] of groups) {
    if (checked++ >= MAX_NEW_GROUPS_PER_RUN) break;
    const older = await pbAdmin.searchClientErrors({ groupHash, to: windowStart, perPage: 1 });
    if (older.totalItems === 0) {
      await fireAlert({
        kind: "client_error_new_group",
        dim: groupHash,
        message: info.message,
        value: info.count,
        title: msg`New client error`,
        body: msg`A new error appeared: ${info.message.slice(0, 120)}`,
      });
    }
  }

  const failedActivities = await pbAdmin.searchActivities({
    success: false,
    perPage: 200,
    sortBy: "-created",
  });
  const failuresByFn = new Map<string, number>();
  for (const activity of failedActivities.items) {
    if (activity.created < windowStart) continue;
    if (activity.error?.startsWith("FORBIDDEN")) {
      failuresByFn.set("__denied__", (failuresByFn.get("__denied__") ?? 0) + 1);
      continue;
    }
    failuresByFn.set(activity.name, (failuresByFn.get(activity.name) ?? 0) + 1);
  }

  for (const [name, count] of failuresByFn) {
    if (name === "__denied__") continue;
    if (count >= fnFailureThreshold()) {
      await fireAlert({
        kind: "server_fn_failures",
        dim: name,
        message: `${name} failed ${count} times in 15 minutes`,
        value: count,
        title: msg`Server function failing`,
        body: msg`${name} failed ${count} times in the last 15 minutes`,
      });
    }
  }

  const deniedCount = failuresByFn.get("__denied__") ?? 0;
  if (deniedCount >= 1) {
    await fireAlert({
      kind: "denied_access",
      dim: "",
      message: `${deniedCount} denied admin access attempts in 15 minutes`,
      value: deniedCount,
      title: msg`Denied access attempt`,
      body: msg`${deniedCount} denied admin access attempts in the last 15 minutes`,
    });
  }
};