feat(telemetry): rollups, alerts, scheduler, health checks, admin server fns
- pure rollup accumulators with tests; daily upserts keyed date/metric/dim - alert checks with db-backed cooldown and push to admin players - in-process scheduler started from health probes - health endpoint checks pb + supertokens, stale-while-revalidate, always 200 - sse connection counter moved to emitter - telemetry feature server fns: events/errors/groups/rollups/alerts/status
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
import { Logger } from "@/lib/logger";
|
||||
import { runScheduledRollups } from "./rollup.server";
|
||||
import { runAlertChecks } from "./alerts.server";
|
||||
|
||||
const logger = new Logger("Telemetry");
|
||||
|
||||
const ROLLUP_INTERVAL_MS = 15 * 60 * 1000;
|
||||
const ALERT_INTERVAL_MS = 5 * 60 * 1000;
|
||||
|
||||
interface SchedulerStatus {
|
||||
running: boolean;
|
||||
lastRollupAt?: string;
|
||||
lastAlertCheckAt?: string;
|
||||
lastRollupError?: string;
|
||||
lastAlertError?: string;
|
||||
}
|
||||
|
||||
const status: SchedulerStatus = { running: false };
|
||||
|
||||
let initialized = false;
|
||||
let rollupInFlight = false;
|
||||
let alertsInFlight = false;
|
||||
let lastRollupDay: string | undefined;
|
||||
|
||||
const tickRollups = async () => {
|
||||
if (rollupInFlight) return;
|
||||
rollupInFlight = true;
|
||||
try {
|
||||
lastRollupDay = await runScheduledRollups(lastRollupDay);
|
||||
status.lastRollupAt = new Date().toISOString();
|
||||
status.lastRollupError = undefined;
|
||||
} catch (error) {
|
||||
status.lastRollupError = error instanceof Error ? error.message : String(error);
|
||||
logger.error("Rollup run failed", error);
|
||||
} finally {
|
||||
rollupInFlight = false;
|
||||
}
|
||||
};
|
||||
|
||||
const tickAlerts = async () => {
|
||||
if (alertsInFlight) return;
|
||||
alertsInFlight = true;
|
||||
try {
|
||||
await runAlertChecks();
|
||||
status.lastAlertCheckAt = new Date().toISOString();
|
||||
status.lastAlertError = undefined;
|
||||
} catch (error) {
|
||||
status.lastAlertError = error instanceof Error ? error.message : String(error);
|
||||
logger.error("Alert check failed", error);
|
||||
} finally {
|
||||
alertsInFlight = false;
|
||||
}
|
||||
};
|
||||
|
||||
// Single-replica deployment makes in-process intervals a valid scheduler
|
||||
// (same precedent as the badge migration job). Started lazily from request
|
||||
// handlers (health probes guarantee a call within seconds of boot).
|
||||
export const ensureTelemetryScheduler = () => {
|
||||
if (initialized || typeof window !== "undefined") return;
|
||||
initialized = true;
|
||||
status.running = true;
|
||||
|
||||
setTimeout(() => void tickRollups(), 15_000);
|
||||
setTimeout(() => void tickAlerts(), 30_000);
|
||||
setInterval(() => void tickRollups(), ROLLUP_INTERVAL_MS);
|
||||
setInterval(() => void tickAlerts(), ALERT_INTERVAL_MS);
|
||||
|
||||
logger.info("Telemetry scheduler started");
|
||||
};
|
||||
|
||||
export const getSchedulerStatus = (): SchedulerStatus => ({ ...status });
|
||||
Reference in New Issue
Block a user