feat(webapp): add lightweight external OIDC telemetry

Mono-RevId: a8ecd733ed7496919dee9d72f010d949bafe9514

Author: carderne

apps/webapp/app/routes/$organizationSlug.[.well-known].jwks[.]json.ts

@@ -1,33 +1,36 @@
 import { json, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { env } from "~/env.server";
+import { withOidcMetadataTelemetry } from "~/services/oidc/externalOidcTelemetry.server";
 import { logger } from "~/services/logger.server";
 import { externalOidcSigningService } from "~/services/oidc/instance.server";
 import { OIDC_JWKS_HTTP_MAX_AGE_SECONDS } from "~/services/oidc/constants";
 
-export async function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method !== "GET" && request.method !== "HEAD") {
-    return new Response(null, { status: 405, headers: { Allow: "GET, HEAD" } });
-  }
-  if (!env.OIDC_ISSUER_URL || !params.organizationSlug) {
-    return new Response(null, { status: 404, headers: { "Cache-Control": "no-store" } });
-  }
+export function loader({ request, params }: LoaderFunctionArgs) {
+  return withOidcMetadataTelemetry("jwks", async () => {
+    if (request.method !== "GET" && request.method !== "HEAD") {
+      return new Response(null, { status: 405, headers: { Allow: "GET, HEAD" } });
+    }
+    if (!env.OIDC_ISSUER_URL || !params.organizationSlug) {
+      return new Response(null, { status: 404, headers: { "Cache-Control": "no-store" } });
+    }
 
-  try {
-    const jwks = await externalOidcSigningService.getPublicJwks();
-    return json(jwks, {
-      headers: {
-        "Cache-Control": `public, max-age=${OIDC_JWKS_HTTP_MAX_AGE_SECONDS}`,
-        "Access-Control-Allow-Origin": "*",
-      },
-    });
-  } catch {
-    logger.error("Unable to load OIDC public keys");
-    return json(
-      { error: "OIDC public keys unavailable" },
-      {
-        status: 503,
-        headers: { "Cache-Control": "no-store" },
-      }
-    );
-  }
+    try {
+      const jwks = await externalOidcSigningService.getPublicJwks();
+      return json(jwks, {
+        headers: {
+          "Cache-Control": `public, max-age=${OIDC_JWKS_HTTP_MAX_AGE_SECONDS}`,
+          "Access-Control-Allow-Origin": "*",
+        },
+      });
+    } catch {
+      logger.error("Unable to load OIDC public keys");
+      return json(
+        { error: "OIDC public keys unavailable" },
+        {
+          status: 503,
+          headers: { "Cache-Control": "no-store" },
+        }
+      );
+    }
+  });
 }

apps/webapp/app/routes/$organizationSlug.[.well-known].openid-configuration.ts

@@ -1,20 +1,23 @@
 import { json, type LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { env } from "~/env.server";
+import { withOidcMetadataTelemetry } from "~/services/oidc/externalOidcTelemetry.server";
 import { OIDC_JWKS_HTTP_MAX_AGE_SECONDS } from "~/services/oidc/constants";
 import { oidcDiscovery } from "~/utils/oidcIssuer";
 
 export function loader({ request, params }: LoaderFunctionArgs) {
-  if (request.method !== "GET" && request.method !== "HEAD") {
-    return new Response(null, { status: 405, headers: { Allow: "GET, HEAD" } });
-  }
-  if (!env.OIDC_ISSUER_URL || !params.organizationSlug) {
-    return new Response(null, { status: 404, headers: { "Cache-Control": "no-store" } });
-  }
+  return withOidcMetadataTelemetry("discovery", () => {
+    if (request.method !== "GET" && request.method !== "HEAD") {
+      return new Response(null, { status: 405, headers: { Allow: "GET, HEAD" } });
+    }
+    if (!env.OIDC_ISSUER_URL || !params.organizationSlug) {
+      return new Response(null, { status: 404, headers: { "Cache-Control": "no-store" } });
+    }
 
-  return json(oidcDiscovery(env.OIDC_ISSUER_URL, params.organizationSlug), {
-    headers: {
-      "Cache-Control": `public, max-age=${OIDC_JWKS_HTTP_MAX_AGE_SECONDS}`,
-      "Access-Control-Allow-Origin": "*",
-    },
+    return json(oidcDiscovery(env.OIDC_ISSUER_URL, params.organizationSlug), {
+      headers: {
+        "Cache-Control": `public, max-age=${OIDC_JWKS_HTTP_MAX_AGE_SECONDS}`,
+        "Access-Control-Allow-Origin": "*",
+      },
+    });
   });
 }

apps/webapp/app/services/oidc/externalOidc.server.ts

@@ -1,4 +1,3 @@
-import { getMeter } from "@internal/tracing";
 import { RunId } from "@trigger.dev/core/v3/isomorphic";
 import type {
   ExternalOidcController,
@@ -126,9 +125,3 @@ export function getExternalOidcRateLimits(): ExternalOidcRateLimits {
       )
   );
 }
-
-export const externalOidcOutcomes = singleton("externalOidcOutcomes", () =>
-  getMeter("external-oidc").createCounter("external_oidc.token.requests", {
-    description: "External OIDC token requests by outcome",
-  })
-);

apps/webapp/app/services/oidc/externalOidcTelemetry.server.ts

@@ -0,0 +1,46 @@
+import { getMeter } from "@internal/tracing";
+import { singleton } from "~/utils/singleton";
+
+const DURATION_BUCKETS_MS = [1, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000];
+
+export function externalOidcTelemetry() {
+  return singleton("externalOidcTelemetry", () => {
+    const meter = getMeter("external-oidc");
+    return {
+      tokenRequests: meter.createCounter("external_oidc.token.requests", {
+        description: "External OIDC token requests by outcome",
+      }),
+      tokenDuration: meter.createHistogram("external_oidc.token.duration", {
+        description: "External OIDC token request duration by outcome",
+        unit: "ms",
+        advice: { explicitBucketBoundaries: DURATION_BUCKETS_MS },
+      }),
+      metadataDuration: meter.createHistogram("external_oidc.metadata.duration", {
+        description: "External OIDC metadata request duration and count",
+        unit: "ms",
+        advice: { explicitBucketBoundaries: DURATION_BUCKETS_MS },
+      }),
+    };
+  });
+}
+
+export async function withOidcMetadataTelemetry(
+  endpoint: "jwks" | "discovery",
+  fn: () => Response | Promise<Response>
+): Promise<Response> {
+  const startedAt = performance.now();
+  let status = 500;
+  try {
+    const response = await fn();
+    status = response.status;
+    return response;
+  } catch (error) {
+    if (error instanceof Response) status = error.status;
+    throw error;
+  } finally {
+    externalOidcTelemetry().metadataDuration.record(performance.now() - startedAt, {
+      endpoint,
+      status_class: `${Math.floor(status / 100)}xx`,
+    });
+  }
+}

apps/webapp/app/services/oidc/externalOidcTokenAction.server.ts

@@ -1,15 +1,13 @@
+import { trace } from "@internal/tracing";
 import { json, type ActionFunctionArgs } from "@remix-run/node";
 import { decodeJwt } from "jose";
 import { z } from "zod";
 import { env } from "~/env.server";
 import { logger } from "~/services/logger.server";
-import {
-  externalOidcOutcomes,
-  getExternalOidcController,
-  getExternalOidcRateLimits,
-} from "./externalOidc.server";
+import { getExternalOidcController, getExternalOidcRateLimits } from "./externalOidc.server";
 import { internalRunSigningService } from "./instance.server";
-import { isRunTokenFormat, verifyRunToken } from "./runTokenVerify";
+import { externalOidcTelemetry } from "./externalOidcTelemetry.server";
+import { isRunTokenFormat, verifyRunToken, type RunTokenClaims } from "./runTokenVerify";
 
 const MAX_AUDIENCE_LENGTH = 2_048;
 
@@ -26,35 +24,40 @@ const TokenRequest = z
 
 const NO_STORE = { "Cache-Control": "no-store" };
 
-function failure(
-  code: string,
-  error: string,
-  status: number,
-  headers: Record<string, string> = {}
-) {
-  externalOidcOutcomes.add(1, { outcome: code });
-  return json({ error, code }, { status, headers: { ...NO_STORE, ...headers } });
-}
-
 export async function externalOidcTokenAction({ request }: ActionFunctionArgs) {
-  if (request.method !== "POST") {
-    return failure("METHOD_NOT_ALLOWED", "Method not allowed", 405, { Allow: "POST" });
-  }
+  const startedAt = performance.now();
+  const span = trace.getActiveSpan();
+  let outcome = "OIDC_SERVICE_UNAVAILABLE";
+  let claims: RunTokenClaims | undefined;
 
-  const controller = await getExternalOidcController();
-  if (!controller) {
-    return failure(
-      "OIDC_UNAVAILABLE",
-      "External OIDC is only available on Trigger.dev Cloud",
-      503,
-      { "x-should-retry": "false" }
-    );
+  function failure(
+    code: string,
+    error: string,
+    status: number,
+    headers: Record<string, string> = {}
+  ) {
+    outcome = code;
+    return json({ error, code }, { status, headers: { ...NO_STORE, ...headers } });
   }
 
   try {
+    if (request.method !== "POST") {
+      return failure("METHOD_NOT_ALLOWED", "Method not allowed", 405, { Allow: "POST" });
+    }
+
+    const controller = await getExternalOidcController();
+    if (!controller) {
+      return failure(
+        "OIDC_UNAVAILABLE",
+        "External OIDC is only available on Trigger.dev Cloud",
+        503,
+        { "x-should-retry": "false" }
+      );
+    }
+
     // Only run tokens: root keys, PATs, sessions and public JWTs cannot mint.
     const bearer = /^Bearer ([^\s]+)$/i.exec(request.headers.get("Authorization") ?? "")?.[1];
-    const claims =
+    claims =
       bearer && env.RUN_TOKEN_ISSUER_URL && isRunTokenFormat(bearer)
         ? await verifyRunToken({
             token: bearer,
@@ -100,17 +103,40 @@ export async function externalOidcTokenAction({ request }: ActionFunctionArgs) {
 
     const expiry = decodeJwt(result.token).exp;
     if (typeof expiry !== "number") throw new Error("OIDC plugin returned a token without expiry");
-    externalOidcOutcomes.add(1, { outcome: "minted" });
+    outcome = "minted";
     return json(
       { token: result.token, expiresAt: new Date(expiry * 1_000).toISOString() },
       { headers: NO_STORE }
     );
   } catch (error) {
-    logger.error("External OIDC mint failed", { error });
+    logger.error("External OIDC mint failed", {
+      error,
+      organizationId: claims?.org_id,
+      projectId: claims?.project_id,
+      environmentId: claims?.environment_id,
+      runId: claims?.run_id,
+    });
     return failure(
       "OIDC_SERVICE_UNAVAILABLE",
       "OIDC token service is temporarily unavailable",
       503
     );
+  } finally {
+    const telemetry = externalOidcTelemetry();
+    telemetry.tokenRequests.add(1, { outcome });
+    telemetry.tokenDuration.record(performance.now() - startedAt, { outcome });
+    // Reuse the request's sampling decision; tenant IDs never become metric labels.
+    if (span?.isRecording()) {
+      span.setAttributes({
+        "oidc.outcome": outcome,
+        ...(claims && {
+          "oidc.organization_id": claims.org_id,
+          "oidc.project_id": claims.project_id,
+          "oidc.environment_id": claims.environment_id,
+          "oidc.run_id": claims.run_id,
+          "oidc.attempt": claims.attempt,
+        }),
+      });
+    }
   }
 }

apps/webapp/test/externalOidcTelemetry.test.ts

@@ -0,0 +1,61 @@
+import { metrics } from "@opentelemetry/api";
+import {
+  InMemoryMetricExporter,
+  MeterProvider,
+  PeriodicExportingMetricReader,
+} from "@opentelemetry/sdk-metrics";
+import { expect, it } from "vitest";
+import { withOidcMetadataTelemetry } from "../app/services/oidc/externalOidcTelemetry.server";
+
+// Catches: metadata instrumentation swallowing responses/errors or omitting failed requests.
+it("preserves metadata responses and errors while counting their HTTP outcomes", async () => {
+  const exporter = new InMemoryMetricExporter();
+  const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 60_000 });
+  const provider = new MeterProvider({ readers: [reader] });
+  metrics.setGlobalMeterProvider(provider);
+
+  try {
+    const response = new Response("keys", { headers: { "Cache-Control": "public, max-age=300" } });
+    expect(await withOidcMetadataTelemetry("jwks", () => response)).toBe(response);
+
+    const notFound = new Response(null, { status: 404 });
+    await expect(
+      withOidcMetadataTelemetry("discovery", () => {
+        throw notFound;
+      })
+    ).rejects.toBe(notFound);
+
+    const error = new Error("key store unavailable");
+    await expect(
+      withOidcMetadataTelemetry("jwks", () => {
+        throw error;
+      })
+    ).rejects.toBe(error);
+
+    await provider.forceFlush();
+    const points = exporter
+      .getMetrics()
+      .flatMap((resource) =>
+        resource.scopeMetrics.flatMap((scope) =>
+          scope.metrics
+            .filter((metric) => metric.descriptor.name === "external_oidc.metadata.duration")
+            .flatMap((metric) => metric.dataPoints)
+        )
+      );
+    expect(
+      points.map((point) => ({
+        attributes: point.attributes,
+        count: typeof point.value === "object" ? point.value.count : undefined,
+      }))
+    ).toEqual(
+      expect.arrayContaining([
+        { attributes: { endpoint: "jwks", status_class: "2xx" }, count: 1 },
+        { attributes: { endpoint: "discovery", status_class: "4xx" }, count: 1 },
+        { attributes: { endpoint: "jwks", status_class: "5xx" }, count: 1 },
+      ])
+    );
+  } finally {
+    await provider.shutdown();
+    metrics.disable();
+  }
+});