feat(telemetry): rollups, alerts, scheduler, health checks, admin server fns
- pure rollup accumulators with tests; daily upserts keyed date/metric/dim - alert checks with db-backed cooldown and push to admin players - in-process scheduler started from health probes - health endpoint checks pb + supertokens, stale-while-revalidate, always 200 - sse connection counter moved to emitter - telemetry feature server fns: events/errors/groups/rollups/alerts/status
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
import { useServerSuspenseQuery } from "@/lib/tanstack-query/hooks";
|
||||
import {
|
||||
getClientErrorGroups,
|
||||
getRollupRange,
|
||||
getTelemetryRuntimeStatus,
|
||||
listTelemetryAlerts,
|
||||
searchClientErrors,
|
||||
searchClientEvents,
|
||||
} from "./server";
|
||||
import type { ClientErrorSearchParams, ClientEventSearchParams } from "./types";
|
||||
|
||||
interface ErrorGroupParams {
|
||||
from: string;
|
||||
to: string;
|
||||
includeResolved?: boolean;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
interface RollupRangeParams {
|
||||
metrics: string[];
|
||||
from: string;
|
||||
to: string;
|
||||
dim?: string;
|
||||
}
|
||||
|
||||
export const telemetryKeys = {
|
||||
all: ["telemetry"] as const,
|
||||
events: (params: ClientEventSearchParams) => ["telemetry", "events", params] as const,
|
||||
errors: (params: ClientErrorSearchParams) => ["telemetry", "errors", params] as const,
|
||||
errorGroups: (params: ErrorGroupParams) => ["telemetry", "errorGroups", params] as const,
|
||||
rollups: (params: RollupRangeParams) => ["telemetry", "rollups", params] as const,
|
||||
alerts: (page: number) => ["telemetry", "alerts", page] as const,
|
||||
runtimeStatus: ["telemetry", "runtimeStatus"] as const,
|
||||
};
|
||||
|
||||
export const telemetryQueries = {
|
||||
events: (params: ClientEventSearchParams = {}) => ({
|
||||
queryKey: telemetryKeys.events(params),
|
||||
queryFn: () => searchClientEvents({ data: params }),
|
||||
}),
|
||||
errors: (params: ClientErrorSearchParams = {}) => ({
|
||||
queryKey: telemetryKeys.errors(params),
|
||||
queryFn: () => searchClientErrors({ data: params }),
|
||||
}),
|
||||
errorGroups: (params: ErrorGroupParams) => ({
|
||||
queryKey: telemetryKeys.errorGroups(params),
|
||||
queryFn: () => getClientErrorGroups({ data: params }),
|
||||
}),
|
||||
rollups: (params: RollupRangeParams) => ({
|
||||
queryKey: telemetryKeys.rollups(params),
|
||||
queryFn: () => getRollupRange({ data: params }),
|
||||
}),
|
||||
alerts: (page = 1) => ({
|
||||
queryKey: telemetryKeys.alerts(page),
|
||||
queryFn: () => listTelemetryAlerts({ data: { page } }),
|
||||
}),
|
||||
runtimeStatus: () => ({
|
||||
queryKey: telemetryKeys.runtimeStatus,
|
||||
queryFn: () => getTelemetryRuntimeStatus(),
|
||||
}),
|
||||
};
|
||||
|
||||
export const useClientEvents = (params: ClientEventSearchParams = {}) =>
|
||||
useServerSuspenseQuery(telemetryQueries.events(params));
|
||||
|
||||
export const useClientErrors = (params: ClientErrorSearchParams = {}) =>
|
||||
useServerSuspenseQuery(telemetryQueries.errors(params));
|
||||
|
||||
export const useClientErrorGroups = (params: ErrorGroupParams) =>
|
||||
useServerSuspenseQuery(telemetryQueries.errorGroups(params));
|
||||
|
||||
export const useRollupRange = (params: RollupRangeParams) =>
|
||||
useServerSuspenseQuery(telemetryQueries.rollups(params));
|
||||
|
||||
export const useTelemetryRuntimeStatus = () =>
|
||||
useServerSuspenseQuery(telemetryQueries.runtimeStatus());
|
||||
@@ -0,0 +1,171 @@
|
||||
import { createServerFn } from "@tanstack/react-start";
|
||||
import { z } from "zod";
|
||||
import { superTokensAdminFunctionMiddleware } from "@/utils/supertokens";
|
||||
import { pbAdmin } from "@/lib/pocketbase/client";
|
||||
import { toServerResult } from "@/lib/tanstack-query/utils/to-server-result";
|
||||
import {
|
||||
transformClientError,
|
||||
transformClientEvent,
|
||||
} from "@/lib/pocketbase/util/transform-types";
|
||||
import { getSchedulerStatus } from "@/lib/telemetry/scheduler.server";
|
||||
import { getHealthSnapshot } from "@/lib/telemetry/health.server";
|
||||
import { getActiveConnectionCount } from "@/lib/events/emitter";
|
||||
import type {
|
||||
ClientError,
|
||||
ClientErrorGroup,
|
||||
ClientEvent,
|
||||
RollupRow,
|
||||
TelemetryAlert,
|
||||
TelemetryListResult,
|
||||
TelemetryRuntimeStatus,
|
||||
} from "./types";
|
||||
|
||||
const clientEventSearchSchema = z.object({
|
||||
page: z.number().optional(),
|
||||
perPage: z.number().max(200).optional(),
|
||||
name: z.string().optional(),
|
||||
player: z.string().optional(),
|
||||
sessionId: z.string().optional(),
|
||||
path: z.string().optional(),
|
||||
from: z.string().optional(),
|
||||
to: z.string().optional(),
|
||||
sortBy: z.string().optional(),
|
||||
});
|
||||
|
||||
export const searchClientEvents = createServerFn()
|
||||
.validator(clientEventSearchSchema)
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<TelemetryListResult<ClientEvent>>(async () => {
|
||||
const result = await pbAdmin.searchClientEvents(data);
|
||||
return { ...result, items: result.items.map(transformClientEvent) };
|
||||
})
|
||||
);
|
||||
|
||||
const clientErrorSearchSchema = z.object({
|
||||
page: z.number().optional(),
|
||||
perPage: z.number().max(200).optional(),
|
||||
groupHash: z.string().optional(),
|
||||
resolved: z.boolean().optional(),
|
||||
player: z.string().optional(),
|
||||
from: z.string().optional(),
|
||||
to: z.string().optional(),
|
||||
sortBy: z.string().optional(),
|
||||
});
|
||||
|
||||
export const searchClientErrors = createServerFn()
|
||||
.validator(clientErrorSearchSchema)
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<TelemetryListResult<ClientError>>(async () => {
|
||||
const result = await pbAdmin.searchClientErrors(data);
|
||||
return { ...result, items: result.items.map(transformClientError) };
|
||||
})
|
||||
);
|
||||
|
||||
const errorGroupsSchema = z.object({
|
||||
from: z.string(),
|
||||
to: z.string(),
|
||||
includeResolved: z.boolean().optional(),
|
||||
limit: z.number().max(100).optional(),
|
||||
});
|
||||
|
||||
export const getClientErrorGroups = createServerFn()
|
||||
.validator(errorGroupsSchema)
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<ClientErrorGroup[]>(async () => {
|
||||
const { from, to, includeResolved = false, limit = 50 } = data;
|
||||
|
||||
const rollups = await pbAdmin.getRollupRange({
|
||||
metrics: ["client_error.count"],
|
||||
from,
|
||||
to,
|
||||
});
|
||||
|
||||
const countByGroup = new Map<string, number>();
|
||||
for (const rollup of rollups) {
|
||||
if (!rollup.dim) continue;
|
||||
countByGroup.set(rollup.dim, (countByGroup.get(rollup.dim) ?? 0) + rollup.value);
|
||||
}
|
||||
|
||||
const topGroups = [...countByGroup.entries()]
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, limit);
|
||||
|
||||
const groups: ClientErrorGroup[] = [];
|
||||
for (const [groupHash, count] of topGroups) {
|
||||
const latest = await pbAdmin.searchClientErrors({ groupHash, perPage: 1 });
|
||||
const sample = latest.items[0];
|
||||
if (!sample) continue;
|
||||
|
||||
// A group counts as resolved when its most recent occurrence is
|
||||
// resolved; a recurrence arrives resolved=false and surfaces again.
|
||||
const resolved = !!sample.resolved;
|
||||
if (resolved && !includeResolved) continue;
|
||||
|
||||
groups.push({
|
||||
group_hash: groupHash,
|
||||
count,
|
||||
lastSeen: sample.created,
|
||||
resolved,
|
||||
sample: {
|
||||
message: sample.message,
|
||||
stack: sample.stack,
|
||||
path: sample.path,
|
||||
source: sample.source,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return groups;
|
||||
})
|
||||
);
|
||||
|
||||
export const resolveClientErrorGroup = createServerFn({ method: "POST" })
|
||||
.validator(z.object({ groupHash: z.string(), resolved: z.boolean().optional() }))
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<{ updated: number }>(async () => {
|
||||
const updated = await pbAdmin.resolveErrorsByGroup(data.groupHash, data.resolved ?? true);
|
||||
return { updated };
|
||||
})
|
||||
);
|
||||
|
||||
const rollupRangeSchema = z.object({
|
||||
metrics: z.array(z.string()).min(1).max(8),
|
||||
from: z.string().regex(/^\d{4}-\d{2}-\d{2}$/),
|
||||
to: z.string().regex(/^\d{4}-\d{2}-\d{2}$/),
|
||||
dim: z.string().optional(),
|
||||
});
|
||||
|
||||
export const getRollupRange = createServerFn()
|
||||
.validator(rollupRangeSchema)
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<RollupRow[]>(async () => pbAdmin.getRollupRange(data))
|
||||
);
|
||||
|
||||
export const listTelemetryAlerts = createServerFn()
|
||||
.validator(
|
||||
z.object({ page: z.number().optional(), perPage: z.number().max(200).optional() })
|
||||
)
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async ({ data }) =>
|
||||
toServerResult<TelemetryListResult<TelemetryAlert>>(async () =>
|
||||
pbAdmin.listTelemetryAlerts(data.page, data.perPage)
|
||||
)
|
||||
);
|
||||
|
||||
export const getTelemetryRuntimeStatus = createServerFn()
|
||||
.middleware([superTokensAdminFunctionMiddleware])
|
||||
.handler(async () =>
|
||||
toServerResult<TelemetryRuntimeStatus>(async () => {
|
||||
const health = await getHealthSnapshot();
|
||||
return {
|
||||
health: { status: health.status, checks: health.checks },
|
||||
scheduler: getSchedulerStatus(),
|
||||
activeSseConnections: getActiveConnectionCount(),
|
||||
};
|
||||
})
|
||||
);
|
||||
@@ -0,0 +1,38 @@
|
||||
export type {
|
||||
ClientEventRecord as ClientEvent,
|
||||
ClientErrorRecord as ClientError,
|
||||
ClientErrorSource,
|
||||
ClientEventSearchParams,
|
||||
ClientErrorSearchParams,
|
||||
RollupRecord as RollupRow,
|
||||
TelemetryAlertRecord as TelemetryAlert,
|
||||
TelemetryListResult,
|
||||
} from "@/lib/pocketbase/services/telemetry";
|
||||
|
||||
export interface ClientErrorGroup {
|
||||
group_hash: string;
|
||||
count: number;
|
||||
lastSeen: string;
|
||||
resolved: boolean;
|
||||
sample: {
|
||||
message: string;
|
||||
stack?: string;
|
||||
path?: string;
|
||||
source?: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface TelemetryRuntimeStatus {
|
||||
health: {
|
||||
status: "ok" | "degraded";
|
||||
checks: { pocketbase: string; supertokens: string };
|
||||
};
|
||||
scheduler: {
|
||||
running: boolean;
|
||||
lastRollupAt?: string;
|
||||
lastAlertCheckAt?: string;
|
||||
lastRollupError?: string;
|
||||
lastAlertError?: string;
|
||||
};
|
||||
activeSseConnections: number;
|
||||
}
|
||||
Reference in New Issue
Block a user