feat(observability): OpenTelemetry tracing + log/trace correlation

Adds OTel auto-instrumentation (http, express, prisma/pg, redis, ...) exporting
OTLP to Tempo, so iios-service requests appear as distributed traces in Grafana.

observability/tracing.ts is imported FIRST in main.ts — auto-instrumentation
patches modules as they are require()d, so anything loaded earlier is never
traced. Env-driven (OTEL_SERVICE_NAME / OTEL_EXPORTER_OTLP_ENDPOINT); with the
endpoint unset the SDK never starts, so dev and tests carry zero overhead.
Health/metrics probes are excluded — they would otherwise swamp Tempo and bury
real request traces.

The existing P9 trace middleware now adopts the ACTIVE OTel trace id, so an
http.request log line and its distributed trace share one id (Loki -> Tempo
pivot in Grafana), falling back to x-trace-id / traceparent / generated.

pnpm-lock.yaml regenerated — CI and the Dockerfile both use
'pnpm install --frozen-lockfile', which would hard-fail on an unchanged lock.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.github/workflows/deploy-iios-service.yml

@@ -42,6 +42,7 @@ jobs:
 
       - name: Bump k8s-pods image tag (ArgoCD deploys)
         run: |
+          rm -rf /tmp/kp   # dind-builder is a persistent host: clear any stale checkout from a prior run
           git clone --depth 1 -b main \
             "https://mcp-bot:${{ secrets.K8S_PODS_TOKEN }}@git.lynkedup.cloud/platform-engineering/k8s-pods.git" /tmp/kp
           cd /tmp/kp

packages/iios-service/package.json

@@ -30,7 +30,10 @@
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.2",
     "socket.io": "^4.8.3",
-    "web-push": "^3.6.7"
+    "web-push": "^3.6.7",
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/sdk-node": "^0.220.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.78.0"
   },
   "devDependencies": {
     "@insignia/iios-testkit": "workspace:*",

packages/iios-service/src/main.ts

@@ -1,3 +1,4 @@
+import './observability/tracing'; // MUST be first — auto-instrumentation patches modules on require
 import 'dotenv/config';
 import 'reflect-metadata';
 import { NestFactory } from '@nestjs/core';

packages/iios-service/src/observability/trace.middleware.ts

@@ -1,4 +1,5 @@
 import type { Request, Response, NextFunction } from 'express';
+import { trace } from '@opentelemetry/api';
 import { runWithTrace, newTraceId, traceIdFromTraceparent } from './trace-context';
 import { logJson } from './logger';
 
@@ -9,8 +10,13 @@ import { logJson } from './logger';
  * the response finishes (method, path, status, ms). Wired via app.use() in main.ts.
  */
 export function traceMiddleware(req: Request, res: Response, next: NextFunction): void {
+  // Prefer the ACTIVE OpenTelemetry trace id: that is what makes this log line
+  // joinable to its distributed trace in Grafana (Loki -> Tempo). Falls back to
+  // the inbound header, then traceparent, then a generated id, so a correlation
+  // id is always present even when tracing is disabled.
+  const otelTraceId = trace.getActiveSpan()?.spanContext().traceId;
   const headerTrace = (req.headers['x-trace-id'] as string | undefined)?.trim();
-  const traceId = headerTrace || traceIdFromTraceparent(req.headers['traceparent'] as string | undefined) || newTraceId();
+  const traceId = otelTraceId || headerTrace || traceIdFromTraceparent(req.headers['traceparent'] as string | undefined) || newTraceId();
   res.setHeader('x-trace-id', traceId);
   const startedAt = Date.now();
   runWithTrace(traceId, () => {

packages/iios-service/src/observability/tracing.ts

@@ -0,0 +1,50 @@
+/**
+ * OpenTelemetry bootstrap. MUST be imported before anything else in main.ts —
+ * auto-instrumentation works by patching modules (http, express, pg, redis, …)
+ * as they are require()d, so any module loaded before this runs is never traced.
+ *
+ * Env-driven on purpose: the OTel SDK reads OTEL_SERVICE_NAME,
+ * OTEL_EXPORTER_OTLP_ENDPOINT and OTEL_EXPORTER_OTLP_PROTOCOL itself, so the
+ * collector target is a deployment concern rather than a code change. When
+ * OTEL_EXPORTER_OTLP_ENDPOINT is unset the SDK never starts, so local dev and
+ * tests run with zero tracing overhead and no exporter errors.
+ *
+ * Traces land in Tempo and are viewable in Grafana. The existing P9 trace
+ * middleware adopts the active OTel trace id, so an `http.request` log line and
+ * its distributed trace share one id (Loki -> Tempo pivot).
+ */
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+
+const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+
+if (endpoint) {
+  const sdk = new NodeSDK({
+    instrumentations: [
+      getNodeAutoInstrumentations({
+        // Noisy and low value: every file read becomes a span.
+        '@opentelemetry/instrumentation-fs': { enabled: false },
+        // k8s probes hit /health constantly; tracing them would swamp Tempo
+        // and bury the real request traces.
+        '@opentelemetry/instrumentation-http': {
+          ignoreIncomingRequestHook: (req) => {
+            const url = req.url ?? '';
+            return ['/health', '/ready', '/healthz', '/readyz', '/metrics'].some((p) =>
+              url.startsWith(p),
+            );
+          },
+        },
+      }),
+    ],
+  });
+
+  sdk.start();
+
+  const shutdown = (): void => {
+    // Flush buffered spans before exit, else the last requests before a
+    // rollout are lost.
+    void sdk.shutdown().finally(() => process.exit(0));
+  };
+  process.on('SIGTERM', shutdown);
+  process.on('SIGINT', shutdown);
+}