Refactor code structure for improved readability and maintainability

apps/web/src/app/api/ai/classify/route.ts

@@ -0,0 +1,57 @@
+import { type NextRequest, NextResponse } from 'next/server';
+
+import { RunpodError } from '../../../../lib/runpod/client';
+import { rpDetect } from '../../../../lib/runpod/endpoints';
+
+export const runtime = 'nodejs';
+export const maxDuration = 60;
+
+const MAX_BASE64 = 4_000_000; // ~3 MB decoded — under serverless body limits
+
+/**
+ * Object-detection proxy for the RunPod YOLO construction-material classifier (#1).
+ * The key + endpoint URL stay server-side. The client (runpodYoloProvider) calls
+ * this only when NEXT_PUBLIC_APG_RUNPOD_DETECT is on; otherwise detection runs
+ * fully in-browser (COCO-SSD) with no server round-trip. Returns the SDK's
+ * DetectedObject[] shape (box as 0..1 fractions) so it drops straight into the
+ * existing Objects browser / smart albums / search.
+ */
+export async function POST(req: NextRequest) {
+  if (!process.env.RUNPOD_API_KEY || !process.env.RUNPOD_YOLO_URL) {
+    return NextResponse.json(
+      { error: 'RunPod detection is not configured (set RUNPOD_API_KEY + RUNPOD_YOLO_URL).' },
+      { status: 503 },
+    );
+  }
+
+  let body: unknown;
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid request body.' }, { status: 400 });
+  }
+
+  const { imageBase64, width, height } = (body ?? {}) as {
+    imageBase64?: unknown;
+    width?: unknown;
+    height?: unknown;
+  };
+  if (typeof imageBase64 !== 'string' || imageBase64.length === 0 || imageBase64.length > MAX_BASE64) {
+    return NextResponse.json({ error: 'Invalid or oversized image.' }, { status: 400 });
+  }
+  const w = Number(width);
+  const h = Number(height);
+
+  try {
+    const objects = await rpDetect(
+      imageBase64,
+      Number.isFinite(w) && w > 0 ? w : 1,
+      Number.isFinite(h) && h > 0 ? h : 1,
+    );
+    return NextResponse.json({ objects });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Detection failed.';
+    const status = err instanceof RunpodError ? err.status : 502;
+    return NextResponse.json({ error: message }, { status });
+  }
+}

apps/web/src/app/api/ai/denoise/route.ts

@@ -0,0 +1,40 @@
+import { type NextRequest, NextResponse } from 'next/server';
+
+import { RunpodError } from '../../../../lib/runpod/client';
+import { rpDenoiseAudio } from '../../../../lib/runpod/endpoints';
+
+export const runtime = 'nodejs';
+export const maxDuration = 60; // cold-start denoise worker can take a while
+
+/**
+ * Audio noise-removal proxy. Accepts base64 WAV (48 kHz mono PCM16, produced
+ * in-browser by lib/audioCapture) and returns a cleaned base64 WAV. Calls the
+ * RunPod audio-denoise endpoint (RUNPOD_AUDIO_DENOISE_URL) — key stays
+ * server-side. Used before transcription on noisy sites. See docs/AI-SETUP.md.
+ */
+
+const MAX_BASE64 = 12_000_000; // ~9 MB decoded WAV
+
+export async function POST(req: NextRequest) {
+  let body: { audio?: unknown };
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON body.' }, { status: 400 });
+  }
+
+  const audio = typeof body.audio === 'string' ? body.audio : '';
+  if (!audio) return NextResponse.json({ error: 'Missing audio.' }, { status: 400 });
+  if (audio.length > MAX_BASE64) {
+    return NextResponse.json({ error: 'Audio too long — keep it under ~30s.' }, { status: 413 });
+  }
+
+  try {
+    const { audioB64 } = await rpDenoiseAudio(audio);
+    return NextResponse.json({ audio: audioB64 });
+  } catch (e) {
+    const msg =
+      e instanceof RunpodError ? e.message : e instanceof Error ? e.message : 'Denoise failed.';
+    return NextResponse.json({ error: msg }, { status: 502 });
+  }
+}

apps/web/src/app/api/ai/edit/route.ts

@@ -1,5 +1,8 @@
 import { type NextRequest, NextResponse } from 'next/server';
 
+import { RunpodError } from '../../../../lib/runpod/client';
+import { rpImg2Img, rpInpaint, rpRemoveBackground, rpUpscale } from '../../../../lib/runpod/endpoints';
+
 export const runtime = 'nodejs';
 export const maxDuration = 60; // SD / cold-start models can take a while
 
@@ -7,6 +10,12 @@ export const maxDuration = 60; // SD / cold-start models can take a while
  * Generative image-edit proxy. The BACKEND is pluggable so you are not tied to a
  * paid model — pick one with env `AI_EDIT_PROVIDER` (default `auto`):
  *
+ *   - `runpod`      → your RunPod serverless GPU endpoints (one per model, per
+ *                     docs/model-api-spec). Maps each op → endpoint: restore/upscale
+ *                     → Real-ESRGAN (#7), colorize → DDColor (#8), replace-sky/
+ *                     magic-eraser/generative-fill → SD 3.5 masked inpaint (#9),
+ *                     prompt → SD 3.5 img2img (#10). Env: RUNPOD_API_KEY + per-model
+ *                     RUNPOD_*_URL. Key stays server-side. See docs/AI-SETUP.md.
  *   - `local`       → your own Stable Diffusion server (Automatic1111 / Forge /
  *                     SD.Next img2img API). Env: LOCAL_SD_URL (e.g. http://127.0.0.1:7860).
  *                     100% free + private, needs a GPU box you run. Most reliable.
@@ -14,10 +23,10 @@ export const maxDuration = 60; // SD / cold-start models can take a while
  *                     Env: HF_API_TOKEN (free), HF_IMAGE_MODEL (default instruct-pix2pix).
  *   - `gemini`      → Google Gemini image model (needs a billed key for image output).
  *                     Env: GEMINI_API_KEY, GEMINI_IMAGE_MODEL.
- *   - `auto`        → first configured of: local → huggingface → gemini.
+ *   - `auto`        → first configured of: runpod → local → huggingface → gemini.
  *
- * NOTE: `remove-background` never reaches here — it runs fully in-browser (@imgly,
- * free, no key). Object detection / faces / OCR / embeddings are also 100% in-browser.
+ * NOTE: `remove-background` runs in-browser by default (@imgly, free, no key), so
+ * it usually never reaches here. Object detection uses its own route (/api/ai/classify).
  * See docs/AI-SETUP.md.
  */
 
@@ -31,19 +40,39 @@ const OP_PROMPTS: Record<string, string> = {
 
 const MAX_BASE64 = 4_000_000; // ~3 MB decoded — stays under serverless body limits (e.g. Vercel ~4.5MB)
 
-type Provider = 'local' | 'huggingface' | 'gemini' | 'none';
+type Provider = 'runpod' | 'local' | 'huggingface' | 'gemini' | 'none';
 
 function resolveProvider(): Provider {
   const explicit = (process.env.AI_EDIT_PROVIDER || 'auto').toLowerCase();
-  if (explicit === 'local' || explicit === 'huggingface' || explicit === 'gemini') return explicit;
+  if (
+    explicit === 'runpod' ||
+    explicit === 'local' ||
+    explicit === 'huggingface' ||
+    explicit === 'gemini'
+  )
+    return explicit;
   if (explicit === 'none') return 'none';
-  // auto: prefer a private local server, then free HF, then Gemini.
+  // auto: prefer RunPod GPU endpoints, then a private local server, then free HF, then Gemini.
+  // Detect RunPod when the key + ANY image endpoint URL is set (an upscale/colorize-only
+  // deployment is valid — not just the SD ones).
+  if (
+    process.env.RUNPOD_API_KEY &&
+    (process.env.RUNPOD_SD_IMG2IMG_URL ||
+      process.env.RUNPOD_SD_INPAINT_URL ||
+      process.env.RUNPOD_UPSCALE_URL ||
+      process.env.RUNPOD_COLORIZE_URL ||
+      process.env.RUNPOD_BG_REMOVE_URL)
+  )
+    return 'runpod';
   if (process.env.LOCAL_SD_URL) return 'local';
   if (process.env.HF_API_TOKEN) return 'huggingface';
   if (process.env.GEMINI_API_KEY) return 'gemini';
   return 'none';
 }
 
+/** Ops that only the RunPod (mask/fixed-function) backend can serve. */
+const RUNPOD_ONLY_OPS = new Set(['upscale', 'magic-eraser', 'generative-fill']);
+
 interface EditResult {
   imageBase64: string;
   mimeType: string;
@@ -55,7 +84,7 @@ export async function POST(req: NextRequest) {
     return NextResponse.json(
       {
         error:
-          'AI image editing is not configured. Set LOCAL_SD_URL (own Stable Diffusion), HF_API_TOKEN (free Hugging Face), or GEMINI_API_KEY. Background removal and all analysis still work with no key. See docs/AI-SETUP.md.',
+          'AI image editing is not configured. Set AI_EDIT_PROVIDER=runpod + RUNPOD_API_KEY + the per-model RUNPOD_*_URL vars (RunPod GPU), or LOCAL_SD_URL (own Stable Diffusion), HF_API_TOKEN (free Hugging Face), or GEMINI_API_KEY. Background removal and all analysis still work with no key. See docs/AI-SETUP.md.',
       },
       { status: 503 },
     );
@@ -68,45 +97,136 @@ export async function POST(req: NextRequest) {
     return NextResponse.json({ error: 'Invalid request body.' }, { status: 400 });
   }
 
-  const { imageBase64, mimeType, op } = (body ?? {}) as {
+  const { imageBase64, mimeType, op, maskBase64, params } = (body ?? {}) as {
     imageBase64?: unknown;
     mimeType?: unknown;
-    op?: { type?: string; prompt?: string };
+    op?: { type?: string; prompt?: string; factor?: number };
+    maskBase64?: unknown;
+    params?: unknown;
   };
 
-  if (typeof imageBase64 !== 'string' || imageBase64.length === 0 || imageBase64.length > MAX_BASE64) {
-    return NextResponse.json({ error: 'Invalid or oversized image (try a smaller image).' }, { status: 400 });
+  if (typeof imageBase64 !== 'string' || imageBase64.length === 0) {
+    return NextResponse.json({ error: 'Invalid image.' }, { status: 400 });
+  }
+  const hasMask = typeof maskBase64 === 'string' && maskBase64.length > 0;
+  // Image + mask share one request body — budget them together against the cap.
+  if (imageBase64.length + (hasMask ? (maskBase64 as string).length : 0) > MAX_BASE64) {
+    return NextResponse.json({ error: 'Image (plus mask) is too large — try a smaller image.' }, { status: 400 });
   }
   const safeMime =
     typeof mimeType === 'string' && /^image\/(jpeg|png|webp)$/.test(mimeType) ? mimeType : 'image/jpeg';
 
+  const opType = op?.type ?? '';
+  if (provider !== 'runpod' && RUNPOD_ONLY_OPS.has(opType)) {
+    return NextResponse.json(
+      { error: 'This edit needs the RunPod backend (set AI_EDIT_PROVIDER=runpod).' },
+      { status: 400 },
+    );
+  }
+
   // Build the instruction from an allow-listed op (never trust arbitrary server prompts).
-  let instruction: string;
-  if (op?.type === 'prompt') {
-    const p = typeof op.prompt === 'string' ? op.prompt.trim() : '';
+  let instruction = '';
+  if (opType === 'prompt' || opType === 'generative-fill') {
+    const p = typeof op?.prompt === 'string' ? op.prompt.trim() : '';
     if (!p) return NextResponse.json({ error: 'Empty prompt.' }, { status: 400 });
     instruction = p.slice(0, 500);
-  } else if (op?.type === 'replace-sky' && typeof op.prompt === 'string' && op.prompt.trim()) {
-    instruction = `Replace the sky with: ${op.prompt.trim().slice(0, 300)}. Keep the foreground unchanged and photorealistic.`;
-  } else if (op?.type && OP_PROMPTS[op.type]) {
-    instruction = OP_PROMPTS[op.type]!;
-  } else {
+  } else if (opType === 'replace-sky') {
+    instruction =
+      typeof op?.prompt === 'string' && op.prompt.trim()
+        ? `Replace the sky with: ${op.prompt.trim().slice(0, 300)}. Keep the foreground unchanged and photorealistic.`
+        : OP_PROMPTS['replace-sky']!;
+  } else if (opType === 'magic-eraser') {
+    instruction = 'Fill the selected region with a clean, seamless, plausible background. Photorealistic.';
+  } else if (OP_PROMPTS[opType]) {
+    instruction = OP_PROMPTS[opType]!;
+  } else if (opType !== 'upscale') {
     return NextResponse.json({ error: 'Unsupported operation.' }, { status: 400 });
   }
 
   try {
     let result: EditResult;
-    if (provider === 'local') result = await editLocal(instruction, imageBase64);
+    if (provider === 'runpod')
+      result = await editRunPod(op ?? {}, imageBase64, instruction, hasMask ? (maskBase64 as string) : undefined, params);
+    else if (provider === 'local') result = await editLocal(instruction, imageBase64);
     else if (provider === 'huggingface') result = await editHuggingFace(instruction, imageBase64);
     else result = await editGemini(instruction, imageBase64, safeMime);
     return NextResponse.json(result);
   } catch (err) {
     const message = err instanceof Error ? err.message : 'AI request failed.';
-    const status = err instanceof AiError ? err.status : 502;
+    const status = err instanceof AiError || err instanceof RunpodError ? err.status : 502;
     return NextResponse.json({ error: message }, { status });
   }
 }
 
+// ---------------------------------------------------------------------------
+// Backend: RunPod serverless GPU endpoints (one model per endpoint).
+// Each op maps to the endpoint from the spec; the API key + URLs stay server-side.
+// ---------------------------------------------------------------------------
+interface SdParams {
+  negativePrompt?: string;
+  strength?: number;
+  steps?: number;
+  seed?: number;
+  guidanceScale?: number;
+}
+
+function sanitizeParams(raw: unknown): SdParams {
+  const p = (raw ?? {}) as Record<string, unknown>;
+  const out: SdParams = {};
+  if (typeof p.negativePrompt === 'string' && p.negativePrompt.trim())
+    out.negativePrompt = p.negativePrompt.trim().slice(0, 300);
+  const strength = Number(p.strength);
+  if (Number.isFinite(strength)) out.strength = Math.max(0, Math.min(1, strength));
+  const steps = Number(p.steps);
+  if (Number.isFinite(steps)) out.steps = Math.max(1, Math.min(60, Math.round(steps)));
+  const guidance = Number(p.guidanceScale);
+  if (Number.isFinite(guidance)) out.guidanceScale = Math.max(1, Math.min(20, guidance));
+  const seed = Number(p.seed);
+  if (Number.isFinite(seed)) out.seed = Math.max(0, Math.min(2_147_483_647, Math.round(seed)));
+  return out;
+}
+
+async function editRunPod(
+  op: { type?: string; prompt?: string; factor?: number },
+  imageBase64: string,
+  instruction: string,
+  maskBase64: string | undefined,
+  rawParams: unknown,
+): Promise<EditResult> {
+  const params = sanitizeParams(rawParams);
+  switch (op.type) {
+    case 'remove-background':
+      // U²-Net via rembg (#6) — a real endpoint replacing the flaky in-browser remover.
+      return rpRemoveBackground(imageBase64);
+    case 'restore':
+      // Real-ESRGAN (#7) with the GFPGAN face pass = "Restore & Enhance".
+      return rpUpscale(imageBase64, 4, true);
+    case 'upscale':
+      return rpUpscale(imageBase64, op.factor === 4 ? 4 : 2, false);
+    case 'colorize':
+      // The dedicated DDColor endpoint kept hard-crashing (modelscope). Route
+      // colorize through the img2img model as an instruction instead — reliable,
+      // and high quality once img2img is FLUX. (RUNPOD_COLORIZE_URL now unused.)
+      return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params });
+    case 'prompt':
+      return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params }); // SD 3.5 img2img (#10)
+    case 'replace-sky':
+      // True sky replacement is masked inpaint (#9). Without a mask (no in-app sky
+      // segmentation yet) degrade to a low-strength img2img (#10) so the foreground
+      // is mostly preserved. Add a mask (brush UI / sky-seg) to get real #9.
+      if (maskBase64)
+        return rpInpaint({ imageB64: imageBase64, maskB64: maskBase64, prompt: instruction, ...params });
+      return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params, strength: params.strength ?? 0.4 });
+    case 'magic-eraser':
+    case 'generative-fill':
+      // SD 3.5 masked inpaint (#9) — white in the mask = the region to regenerate.
+      if (!maskBase64) throw new AiError('This edit needs a mask/selection.', 400);
+      return rpInpaint({ imageB64: imageBase64, maskB64: maskBase64, prompt: instruction, ...params });
+    default:
+      throw new AiError('Unsupported operation.', 400);
+  }
+}
+
 class AiError extends Error {
   status: number;
   constructor(message: string, status = 502) {

apps/web/src/app/api/ai/tilt/route.ts

@@ -0,0 +1,44 @@
+import { type NextRequest, NextResponse } from 'next/server';
+
+import { RunpodError } from '../../../../lib/runpod/client';
+import { rpTilt } from '../../../../lib/runpod/endpoints';
+
+export const runtime = 'nodejs';
+export const maxDuration = 60; // cold-start tilt worker can take a while
+
+/**
+ * Camera-tilt estimation proxy. Accepts a base64 image and returns
+ * {rollDegrees, pitchDegrees, fovDegrees} from the RunPod tilt endpoint
+ * (RUNPOD_TILT_URL) so the editor can auto-straighten. Requires the tilt
+ * endpoint to be deployed; errors clearly if RUNPOD_TILT_URL is unset.
+ */
+
+const MAX_BASE64 = 4_000_000; // ~3 MB decoded
+
+export async function POST(req: NextRequest) {
+  let body: { image?: unknown };
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON body.' }, { status: 400 });
+  }
+
+  const image = typeof body.image === 'string' ? body.image : '';
+  if (!image) return NextResponse.json({ error: 'Missing image.' }, { status: 400 });
+  if (image.length > MAX_BASE64) {
+    return NextResponse.json({ error: 'Image too large.' }, { status: 413 });
+  }
+
+  try {
+    const tilt = await rpTilt(image);
+    return NextResponse.json(tilt);
+  } catch (e) {
+    const msg =
+      e instanceof RunpodError
+        ? e.message
+        : e instanceof Error
+          ? e.message
+          : 'Tilt estimate failed.';
+    return NextResponse.json({ error: msg }, { status: 502 });
+  }
+}

apps/web/src/app/api/ai/transcribe/route.ts

@@ -0,0 +1,45 @@
+import { type NextRequest, NextResponse } from 'next/server';
+
+import { RunpodError } from '../../../../lib/runpod/client';
+import { rpTranscribe } from '../../../../lib/runpod/endpoints';
+
+export const runtime = 'nodejs';
+export const maxDuration = 60; // cold-start STT worker can take a while
+
+/**
+ * Speech-to-text proxy for voice annotations. Accepts base64 WAV (16 kHz mono
+ * PCM16, produced in-browser by lib/audioCapture) and returns the transcript.
+ * Calls the RunPod voice-to-text endpoint (RUNPOD_STT_URL) — the key stays
+ * server-side. See docs/AI-SETUP.md.
+ */
+
+const MAX_BASE64 = 8_000_000; // ~6 MB decoded WAV — stays under serverless body limits
+
+export async function POST(req: NextRequest) {
+  let body: { audio?: unknown; language?: unknown };
+  try {
+    body = await req.json();
+  } catch {
+    return NextResponse.json({ error: 'Invalid JSON body.' }, { status: 400 });
+  }
+
+  const audio = typeof body.audio === 'string' ? body.audio : '';
+  const language = typeof body.language === 'string' ? body.language : undefined;
+  if (!audio) return NextResponse.json({ error: 'Missing audio.' }, { status: 400 });
+  if (audio.length > MAX_BASE64) {
+    return NextResponse.json({ error: 'Audio too long — keep it under ~30s.' }, { status: 413 });
+  }
+
+  try {
+    const { transcript, segments } = await rpTranscribe(audio, { language, punctuation: true });
+    return NextResponse.json({ transcript, segments });
+  } catch (e) {
+    const msg =
+      e instanceof RunpodError
+        ? e.message
+        : e instanceof Error
+          ? e.message
+          : 'Transcription failed.';
+    return NextResponse.json({ error: msg }, { status: 502 });
+  }
+}

apps/web/src/lib/ai/createDemoAIProvider.ts

@@ -2,17 +2,24 @@ import type { AIProvider } from '@photo-gallery/sdk';
 
 import { createClipProvider } from './clipProvider';
 import { createFaceProvider } from './faceProvider';
+import { imageToBase64, maskToBase64 } from './imageEncode';
 import { createOCRProvider } from './ocrProvider';
+import { createRunpodYoloProvider } from './runpodYoloProvider';
 import { createTensorflowProvider } from './tensorflowProvider';
 
 /**
- * The demo's AI provider, combining free in-browser capabilities + Gemini:
+ * The demo's AI provider, combining free in-browser capabilities + a
+ * server-proxied generative-edit backend (RunPod / local SD / Hugging Face / Gemini):
  *  - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key)
  *  - face detection + recognition: face-api.js, in-browser → clustered into People
  *  - OCR: tesseract.js, in-browser → searchable text + the Documents album (no key)
  *  - background removal: @imgly, in-browser WASM (no key) — see generativeEdit
- *  - other generative edits: Gemini, proxied through /api/ai/edit so the API key
- *    stays server-side and is never exposed to the browser.
+ *  - other generative edits: proxied through /api/ai/edit (RunPod / local SD /
+ *    Hugging Face / Gemini, chosen by env) so the API key stays server-side.
+ *
+ * Object detection uses the in-browser COCO-SSD by default, or the RunPod YOLO
+ * construction-material classifier (#1) when NEXT_PUBLIC_APG_RUNPOD_DETECT=true
+ * (proxied through /api/ai/classify, with COCO-SSD as the automatic fallback).
  */
 export function createDemoAIProvider(): AIProvider {
   const tf = createTensorflowProvider();
@@ -20,9 +27,15 @@ export function createDemoAIProvider(): AIProvider {
   const ocr = createOCRProvider();
   const clip = createClipProvider();
 
+  // Opt-in server-side detection; defaults to free in-browser COCO-SSD.
+  const detectObjects =
+    process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT === 'true'
+      ? createRunpodYoloProvider(tf).detectObjects
+      : tf.detectObjects;
+
   return {
-    name: 'demo-ai (coco-ssd + face-api + tesseract + clip + gemini)',
-    detectObjects: tf.detectObjects,
+    name: 'demo-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)',
+    detectObjects,
     detectFaces: face.detectFaces,
     ocr: ocr.ocr,
     embedImage: clip.embedImage,
@@ -30,18 +43,91 @@ export function createDemoAIProvider(): AIProvider {
 
     async generativeEdit(item, image, op) {
       // Remove Background runs fully in-browser (free, no key) via @imgly — works
-      // even when Gemini is unavailable. Other ops go through the Gemini route.
+      // even with no backend. Other ops go through the /api/ai/edit route.
       if (op.type === 'remove-background') {
+        // Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser
+        // @imgly if it's off or the request fails, so this always produces a result.
+        if (process.env.NEXT_PUBLIC_APG_RUNPOD_BG === 'true') {
+          try {
+            const { data, mimeType } = imageToBase64(image, 1600);
+            const res = await fetch('/api/ai/edit', {
+              method: 'POST',
+              headers: { 'content-type': 'application/json' },
+              body: JSON.stringify({
+                imageBase64: data,
+                mimeType,
+                op: { type: 'remove-background' },
+                params: {},
+              }),
+            });
+            if (res.ok) {
+              const { imageBase64: out, mimeType: outMime } = (await res.json()) as {
+                imageBase64: string;
+                mimeType?: string;
+              };
+              return base64ToBlob(out, outMime || 'image/png');
+            }
+          } catch {
+            /* fall through to the in-browser remover */
+          }
+        }
         const inputBlob = await canvasBlob(image, 1600);
         const { removeBackground } = await import('@imgly/background-removal');
         return removeBackground(inputBlob, { output: { format: 'image/png' } });
       }
 
-      const { data, mimeType } = imageToBase64(image, 1280);
+      // Outpaint / expand-canvas: pad the image with a neutral border, mark that
+      // border WHITE in the mask, and run it through the same inpaint path as
+      // generative-fill — no extra backend route needed.
+      if (op.type === 'outpaint') {
+        const { imageBase64: padded, maskBase64: border } = padForOutpaint(
+          image,
+          typeof op.factor === 'number' ? op.factor : 1.5,
+        );
+        const outParams: Record<string, unknown> = {
+          strength: typeof op.strength === 'number' ? op.strength : 0.85,
+        };
+        const outRes = await fetch('/api/ai/edit', {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify({
+            imageBase64: padded,
+            mimeType: 'image/png',
+            op: {
+              type: 'generative-fill',
+              prompt:
+                op.prompt ||
+                'Extend and continue the scene naturally, matching lighting, colors and perspective.',
+            },
+            maskBase64: border,
+            params: outParams,
+          }),
+        });
+        if (!outRes.ok) {
+          const err = (await outRes.json().catch(() => ({}))) as { error?: string };
+          throw new Error(err.error || `AI request failed (${outRes.status}).`);
+        }
+        const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string };
+        return base64ToBlob(outJson.imageBase64, outJson.mimeType || 'image/png');
+      }
+
+      const { data, mimeType, width, height } = imageToBase64(image, 1280);
+      // SD 3.5 masked ops (#9) carry an ImageData mask — rasterize it to a PNG
+      // matched to the (downscaled) image dims, and strip it from the op since
+      // ImageData is not JSON-serializable.
+      const maskBase64 = 'mask' in op ? maskToBase64(op.mask, width, height) : undefined;
+      const wireOp: Record<string, unknown> = { type: op.type };
+      if ('prompt' in op && typeof op.prompt === 'string') wireOp.prompt = op.prompt;
+      if ('factor' in op && typeof op.factor === 'number') wireOp.factor = op.factor;
+
+      // Forward the "edit strength" slider (0..1) so the backend can scale the edit.
+      const params: Record<string, unknown> = {};
+      if ('strength' in op && typeof op.strength === 'number') params.strength = op.strength;
+
       const res = await fetch('/api/ai/edit', {
         method: 'POST',
         headers: { 'content-type': 'application/json' },
-        body: JSON.stringify({ imageBase64: data, mimeType, op }),
+        body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }),
       });
       if (!res.ok) {
         const err = (await res.json().catch(() => ({}))) as { error?: string };
@@ -53,27 +139,60 @@ export function createDemoAIProvider(): AIProvider {
       };
       return base64ToBlob(imageBase64, outMime || 'image/png');
     },
-  };
-}
 
-/** Draw an image to a canvas (downscaled) and return base64 JPEG (no data: prefix). */
-function imageToBase64(
-  image: ImageBitmap | HTMLImageElement,
-  maxDim: number,
-): { data: string; mimeType: string } {
-  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
-  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
-  const scale = Math.min(1, maxDim / Math.max(w, h));
-  const cw = Math.max(1, Math.round(w * scale));
-  const ch = Math.max(1, Math.round(h * scale));
-  const canvas = document.createElement('canvas');
-  canvas.width = cw;
-  canvas.height = ch;
-  const ctx = canvas.getContext('2d');
-  if (!ctx) throw new Error('Canvas not supported.');
-  ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
-  const dataUrl = canvas.toDataURL('image/jpeg', 0.9);
-  return { data: dataUrl.split(',')[1] ?? '', mimeType: 'image/jpeg' };
+    // Voice annotation: record → (optional denoise) → transcribe. Both proxy
+    // through server routes so the RunPod key stays server-side.
+    async transcribeAudio(audioBase64) {
+      const res = await fetch('/api/ai/transcribe', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ audio: audioBase64 }),
+      });
+      if (!res.ok) {
+        const err = (await res.json().catch(() => ({}))) as { error?: string };
+        throw new Error(err.error || `Transcription failed (${res.status}).`);
+      }
+      const { transcript } = (await res.json()) as { transcript?: string };
+      return (transcript ?? '').trim();
+    },
+
+    async denoiseAudio(audioBase64) {
+      const res = await fetch('/api/ai/denoise', {
+        method: 'POST',
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({ audio: audioBase64 }),
+      });
+      if (!res.ok) {
+        const err = (await res.json().catch(() => ({}))) as { error?: string };
+        throw new Error(err.error || `Denoise failed (${res.status}).`);
+      }
+      const { audio } = (await res.json()) as { audio?: string };
+      return audio ?? audioBase64;
+    },
+
+    // Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed);
+    // gate it so the editor's Auto-straighten button only appears when configured.
+    estimateTilt:
+      process.env.NEXT_PUBLIC_APG_RUNPOD_TILT === 'true'
+        ? async (_item, image) => {
+            const { data } = imageToBase64(image, 1024);
+            const res = await fetch('/api/ai/tilt', {
+              method: 'POST',
+              headers: { 'content-type': 'application/json' },
+              body: JSON.stringify({ image: data }),
+            });
+            if (!res.ok) {
+              const err = (await res.json().catch(() => ({}))) as { error?: string };
+              throw new Error(err.error || `Tilt estimate failed (${res.status}).`);
+            }
+            return (await res.json()) as {
+              rollDegrees: number;
+              pitchDegrees: number;
+              fovDegrees: number;
+            };
+          }
+        : undefined,
+  };
 }
 
 /** Draw an image to a canvas (downscaled) and return a JPEG Blob. */
@@ -100,3 +219,55 @@ function base64ToBlob(base64: string, mime: string): Blob {
   for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
   return new Blob([bytes], { type: mime });
 }
+
+/**
+ * Pad an image with a neutral border for outpaint and return {paddedImage, mask}
+ * as base64 PNG — the border is WHITE in the mask (regenerate), the original
+ * image area BLACK (keep). Capped at 1280px on the long side.
+ */
+function padForOutpaint(
+  image: ImageBitmap | HTMLImageElement,
+  factor: number,
+): { imageBase64: string; maskBase64: string } {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const f = Math.max(1.1, Math.min(2, factor));
+  const maxDim = 1280;
+  let pw = Math.round(w * f);
+  let ph = Math.round(h * f);
+  const scale = Math.min(1, maxDim / Math.max(pw, ph));
+  pw = Math.max(16, Math.round(pw * scale));
+  ph = Math.max(16, Math.round(ph * scale));
+  const iw = Math.max(1, Math.round(w * scale));
+  const ih = Math.max(1, Math.round(h * scale));
+  const ox = Math.floor((pw - iw) / 2);
+  const oy = Math.floor((ph - ih) / 2);
+
+  const imgCanvas = document.createElement('canvas');
+  imgCanvas.width = pw;
+  imgCanvas.height = ph;
+  const ictx = imgCanvas.getContext('2d');
+  if (!ictx) throw new Error('Canvas not supported.');
+  // Fill the new border with a blurred, stretched copy of the photo so the model
+  // has real color/context to continue from — flat gray gives it nothing and it
+  // leaves the border gray. Then draw the original sharp on top, centered.
+  ictx.filter = 'blur(28px)';
+  ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph);
+  ictx.filter = 'none';
+  ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih);
+
+  const maskCanvas = document.createElement('canvas');
+  maskCanvas.width = pw;
+  maskCanvas.height = ph;
+  const mctx = maskCanvas.getContext('2d');
+  if (!mctx) throw new Error('Canvas not supported.');
+  mctx.fillStyle = '#ffffff';
+  mctx.fillRect(0, 0, pw, ph);
+  mctx.fillStyle = '#000000';
+  mctx.fillRect(ox, oy, iw, ih);
+
+  return {
+    imageBase64: imgCanvas.toDataURL('image/png').split(',')[1] ?? '',
+    maskBase64: maskCanvas.toDataURL('image/png').split(',')[1] ?? '',
+  };
+}

apps/web/src/lib/ai/imageEncode.ts

@@ -0,0 +1,56 @@
+/**
+ * Shared client-side image encoding helpers for the AI providers. Downscale an
+ * image to a JPEG base64 (keeping the exact output dims), and rasterize a mask
+ * to a PNG base64 matched to those same dims — SD 3.5 (#9) requires the image
+ * and mask to be identical pixel sizes.
+ */
+
+export interface EncodedImage {
+  /** base64 JPEG (no data: prefix). */
+  data: string;
+  mimeType: string;
+  /** Actual pixel dims of the encoded image (after downscale). */
+  width: number;
+  height: number;
+}
+
+/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */
+export function imageToBase64(image: ImageBitmap | HTMLImageElement, maxDim: number): EncodedImage {
+  const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width;
+  const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height;
+  const scale = Math.min(1, maxDim / Math.max(w, h));
+  const cw = Math.max(1, Math.round(w * scale));
+  const ch = Math.max(1, Math.round(h * scale));
+  const canvas = document.createElement('canvas');
+  canvas.width = cw;
+  canvas.height = ch;
+  const ctx = canvas.getContext('2d');
+  if (!ctx) throw new Error('Canvas not supported.');
+  ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch);
+  const dataUrl = canvas.toDataURL('image/jpeg', 0.9);
+  return { data: dataUrl.split(',')[1] ?? '', mimeType: 'image/jpeg', width: cw, height: ch };
+}
+
+/**
+ * Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64
+ * scaled to targetW×targetH so it matches the encoded image exactly.
+ */
+export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string {
+  const tmp = document.createElement('canvas');
+  tmp.width = mask.width;
+  tmp.height = mask.height;
+  const tctx = tmp.getContext('2d');
+  if (!tctx) throw new Error('Canvas not supported.');
+  tctx.putImageData(mask, 0, 0);
+
+  const out = document.createElement('canvas');
+  out.width = targetW;
+  out.height = targetH;
+  const octx = out.getContext('2d');
+  if (!octx) throw new Error('Canvas not supported.');
+  // Nearest-neighbour, not bilinear — keep the mask strictly binary so SD 3.5 gets
+  // crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo.
+  octx.imageSmoothingEnabled = false;
+  octx.drawImage(tmp, 0, 0, targetW, targetH);
+  return out.toDataURL('image/png').split(',')[1] ?? '';
+}

apps/web/src/lib/ai/runpodYoloProvider.ts

@@ -0,0 +1,32 @@
+import type { AIProvider, DetectedObject, MediaItem } from '@photo-gallery/sdk';
+
+import { imageToBase64 } from './imageEncode';
+
+/**
+ * Object detection backed by the RunPod YOLO construction-material classifier
+ * (#1), proxied through /api/ai/classify so the RunPod key stays server-side.
+ *
+ * Opt-in via `NEXT_PUBLIC_APG_RUNPOD_DETECT=true` (see createDemoAIProvider). If
+ * the server call fails (endpoint down / not configured) it falls back to the
+ * given in-browser provider (COCO-SSD), so detection never hard-fails.
+ */
+export function createRunpodYoloProvider(fallback: AIProvider): Pick<AIProvider, 'detectObjects'> {
+  return {
+    async detectObjects(item: MediaItem, image): Promise<DetectedObject[]> {
+      try {
+        const { data, mimeType, width, height } = imageToBase64(image, 1280);
+        const res = await fetch('/api/ai/classify', {
+          method: 'POST',
+          headers: { 'content-type': 'application/json' },
+          body: JSON.stringify({ imageBase64: data, mimeType, width, height }),
+        });
+        if (!res.ok) throw new Error(`classify failed (${res.status})`);
+        const { objects } = (await res.json()) as { objects?: DetectedObject[] };
+        if (Array.isArray(objects)) return objects;
+        throw new Error('classify returned no objects');
+      } catch {
+        return fallback.detectObjects ? fallback.detectObjects(item, image) : [];
+      }
+    },
+  };
+}

apps/web/src/lib/runpod/base64.ts

@@ -0,0 +1,80 @@
+/**
+ * base64 / data-URI helpers shared by the RunPod endpoint wrappers.
+ * Different endpoints name their image field differently and some prefix a
+ * `data:` URI — these helpers normalize both.
+ */
+
+/** "data:image/png;base64,XXXX" -> "XXXX" (leaves a bare base64 string untouched). */
+export function stripDataUri(s: string): string {
+  if (!s.startsWith('data:')) return s;
+  const i = s.indexOf(',');
+  return i === -1 ? s : s.slice(i + 1);
+}
+
+/** Wrap a bare base64 string in a data: URI. */
+export function toDataUri(b64: string, mime: string): string {
+  return `data:${mime};base64,${stripDataUri(b64)}`;
+}
+
+/**
+ * Pull the first image-like base64 string out of a RunPod endpoint's `output`,
+ * regardless of which field name it used. Handles the common shapes:
+ *   "iVBOR..."                       (raw string)
+ *   { image: "..." } / { image_png } / { image_base64 }
+ *   { images: ["..."] }              (array)
+ *   { output: { image: "..." } }     (nested)
+ * Mirrors the A1111 normalization already used for the local SD backend.
+ */
+export function pickOutputImage(output: unknown): string {
+  const s = findImageString(output, false);
+  if (!s) throw new Error('RunPod output did not contain an image.');
+  return stripDataUri(s);
+}
+
+/** Keys whose name implies the value IS the image — any non-empty string is accepted. */
+const IMAGE_KEYS = ['image_png', 'image', 'image_base64', 'images'] as const;
+/** Generic wrapper keys — a string here must actually look like image data. */
+const CONTAINER_KEYS = ['output', 'result', 'data'] as const;
+
+/** A base64 image payload is long; a status/id string ("success", "job-abc") is short. */
+function looksLikeImageData(s: string): boolean {
+  if (s.startsWith('data:image/')) return true;
+  return s.length >= 256 && /^[A-Za-z0-9+/=\s]+$/.test(s.slice(0, 256));
+}
+
+/**
+ * `strict` is true when we descended through a generic wrapper key (output/result/
+ * data), where a bare string could be a status/id rather than an image — so it must
+ * pass `looksLikeImageData`. Under an explicit image key (or at top level) any
+ * non-empty string is taken as the image.
+ */
+function findImageString(value: unknown, strict: boolean, depth = 0): string | undefined {
+  if (depth > 5) return undefined;
+  if (typeof value === 'string') {
+    if (!value) return undefined;
+    return !strict || looksLikeImageData(value) ? value : undefined;
+  }
+  if (Array.isArray(value)) {
+    for (const el of value) {
+      const s = findImageString(el, strict, depth + 1);
+      if (s) return s;
+    }
+    return undefined;
+  }
+  if (value && typeof value === 'object') {
+    const o = value as Record<string, unknown>;
+    for (const key of IMAGE_KEYS) {
+      if (key in o) {
+        const s = findImageString(o[key], false, depth + 1);
+        if (s) return s;
+      }
+    }
+    for (const key of CONTAINER_KEYS) {
+      if (key in o) {
+        const s = findImageString(o[key], true, depth + 1);
+        if (s) return s;
+      }
+    }
+  }
+  return undefined;
+}

apps/web/src/lib/runpod/client.ts

@@ -0,0 +1,121 @@
+/**
+ * Low-level RunPod transport. Every model in the spec runs as its own RunPod
+ * endpoint; this handles the common request envelope, bearer auth, and both
+ * response modes:
+ *
+ *   - `/runsync` (preferred): the job runs synchronously and the body already
+ *     contains `output` (or IS the output for a custom handler).
+ *   - `/run`: returns `{ id, status }`; we poll `/status/{id}` until the job
+ *     reaches a terminal state or the time budget (kept under Vercel's 60s
+ *     function cap) is exhausted.
+ *
+ * RUNPOD_API_KEY is read here so callers never handle the secret directly.
+ */
+
+export class RunpodError extends Error {
+  status: number;
+  constructor(message: string, status = 502) {
+    super(message);
+    this.name = 'RunpodError';
+    this.status = status;
+  }
+}
+
+export interface RunpodCallOpts {
+  /** Human label used in error messages, e.g. "sd-inpaint". */
+  name: string;
+  /** Full endpoint URL from the per-model env var (…/runsync or …/run). */
+  url: string;
+  input: Record<string, unknown>;
+  /** Total budget for the whole call incl. polling. Default 55s (< Vercel 60s). */
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+}
+
+interface RunpodEnvelope {
+  id?: string;
+  status?: string;
+  output?: unknown;
+  error?: unknown;
+}
+
+export async function runpodCall<TOut = unknown>(opts: RunpodCallOpts): Promise<TOut> {
+  const { name, url, input } = opts;
+  const timeoutMs = opts.timeoutMs ?? 55_000;
+  const pollIntervalMs = opts.pollIntervalMs ?? 1500;
+
+  const key = process.env.RUNPOD_API_KEY;
+  if (!key) throw new RunpodError('RUNPOD_API_KEY is not set.', 500);
+
+  const deadline = Date.now() + timeoutMs;
+  const authHeaders = { authorization: `Bearer ${key}` };
+
+  let res: Response;
+  try {
+    res = await fetch(url, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json', ...authHeaders },
+      body: JSON.stringify({ input }),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+  } catch {
+    throw new RunpodError(`Could not reach RunPod (${name}).`, 502);
+  }
+  if (!res.ok) {
+    const detail = (await res.text().catch(() => '')).slice(0, 160);
+    throw new RunpodError(`RunPod ${name} error (${res.status}). ${detail}`.trim(), 502);
+  }
+
+  const data = (await res.json().catch(() => null)) as RunpodEnvelope | null;
+  if (!data || typeof data !== 'object') {
+    throw new RunpodError(`RunPod ${name} returned an invalid response.`, 502);
+  }
+
+  // Terminal failure reported in a job envelope.
+  if (data.status === 'FAILED' || data.status === 'CANCELLED') {
+    throw new RunpodError(`RunPod ${name} job ${data.status.toLowerCase()}.`, 502);
+  }
+  // No job id → this is not an async envelope; the body itself is the output.
+  // Covers custom /runsync handlers that return their result directly, even when
+  // it carries a `status` field (e.g. "success").
+  if (data.id === undefined) {
+    return (data.output !== undefined ? data.output : data) as TOut;
+  }
+  // Job envelope that already carries a completed/inline output.
+  if (data.output !== undefined && (data.status === undefined || data.status === 'COMPLETED')) {
+    return data.output as TOut;
+  }
+
+  // Async: poll /status/{id} until COMPLETED / FAILED / the time budget runs out.
+  // Each wait + fetch is clamped to the remaining budget so the whole call stays
+  // under `timeoutMs` (kept below Vercel's 60s function cap).
+  const statusUrl = url.replace(/\/run(sync)?(\/?)$/, `/status/${data.id}`);
+  for (;;) {
+    const remaining = deadline - Date.now();
+    if (remaining < 500) break; // not enough budget for another round
+    await sleep(Math.min(pollIntervalMs, remaining));
+    const left = deadline - Date.now();
+    if (left <= 0) break;
+    let sres: Response;
+    try {
+      sres = await fetch(statusUrl, {
+        headers: authHeaders,
+        signal: AbortSignal.timeout(Math.min(10_000, Math.max(1000, left))),
+      });
+    } catch {
+      continue; // transient — keep polling until the deadline
+    }
+    if (!sres.ok) continue;
+    const sdata = (await sres.json().catch(() => null)) as RunpodEnvelope | null;
+    if (!sdata) continue;
+    if (sdata.status === 'COMPLETED') return sdata.output as TOut;
+    if (sdata.status === 'FAILED' || sdata.status === 'CANCELLED') {
+      throw new RunpodError(`RunPod ${name} job ${sdata.status.toLowerCase()}.`, 502);
+    }
+  }
+  throw new RunpodError(`RunPod ${name} timed out (raise the endpoint's speed or use /runsync).`, 504);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

apps/web/src/lib/runpod/endpoints.ts

@@ -0,0 +1,283 @@
+/**
+ * Typed wrappers for each RunPod endpoint in `model-api-spec.docx`.
+ * Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input`
+ * contract the spec documents, and normalizes the response.
+ *
+ * Server-side only. Missing/invalid URLs throw a RunpodError(500) so an
+ * unconfigured op surfaces as a clear message in the editor rather than a crash.
+ */
+
+import { stripDataUri, pickOutputImage } from './base64';
+import { RunpodError, runpodCall } from './client';
+import type {
+  Img2ImgReq,
+  InpaintReq,
+  RunpodDetection,
+  RunpodImageResult,
+  TiltResult,
+  TranscriptResult,
+} from './types';
+
+function envNum(v: string | undefined, fallback: number): number {
+  // Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which
+  // would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`.
+  if (v == null || v.trim() === '') return fallback;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : fallback;
+}
+
+function endpointUrl(envVar: string): string {
+  const url = process.env[envVar];
+  if (!url || !/^https?:\/\//i.test(url)) {
+    throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500);
+  }
+  return url;
+}
+
+/** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */
+async function imageOp(
+  name: string,
+  envVar: string,
+  input: Record<string, unknown>,
+): Promise<RunpodImageResult> {
+  const output = await runpodCall({ name, url: endpointUrl(envVar), input });
+  return { imageBase64: pickOutputImage(output), mimeType: 'image/png' };
+}
+
+// ---------------------------------------------------------------------------
+// Image endpoints
+// ---------------------------------------------------------------------------
+
+/** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */
+export function rpRemoveBackground(imageB64: string, model?: string): Promise<RunpodImageResult> {
+  return imageOp('background-removal', 'RUNPOD_BG_REMOVE_URL', {
+    task: 'remove-bg',
+    image: imageB64,
+    ...(model ? { model_name: model } : {}),
+  });
+}
+
+/** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the
+ *  op / restore pass) is authoritative — it is not overridden by any env default. */
+export function rpUpscale(imageB64: string, scale: 2 | 4, faceEnhance = false): Promise<RunpodImageResult> {
+  return imageOp('upscale', 'RUNPOD_UPSCALE_URL', {
+    task: 'upscale',
+    image: imageB64,
+    scale,
+    face_enhance: faceEnhance,
+  });
+}
+
+/** #8 DDColor B&W → colorize. */
+export function rpColorize(imageB64: string, inputSize?: number): Promise<RunpodImageResult> {
+  return imageOp('colorize', 'RUNPOD_COLORIZE_URL', {
+    image: imageB64,
+    ...(inputSize ? { input_size: inputSize } : {}),
+  });
+}
+
+/** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */
+export function rpInpaint(p: InpaintReq): Promise<RunpodImageResult> {
+  const input: Record<string, unknown> = {
+    task: 'inpaint',
+    image: p.imageB64,
+    mask: p.maskB64,
+    prompt: p.prompt,
+    strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8),
+    guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
+    num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
+  };
+  const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
+  if (negative) input.negative_prompt = negative;
+  if (p.seed != null) input.seed = p.seed;
+  return imageOp('sd-inpaint', 'RUNPOD_SD_INPAINT_URL', input);
+}
+
+/** #10 SD 3.5 general prompt edit (img2img, no mask). */
+export function rpImg2Img(p: Img2ImgReq): Promise<RunpodImageResult> {
+  const input: Record<string, unknown> = {
+    task: 'img2img',
+    image: p.imageB64,
+    prompt: p.prompt,
+    strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6),
+    guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
+    num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
+  };
+  const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
+  if (negative) input.negative_prompt = negative;
+  if (p.seed != null) input.seed = p.seed;
+  return imageOp('sd-img2img', 'RUNPOD_SD_IMG2IMG_URL', input);
+}
+
+// ---------------------------------------------------------------------------
+// #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1)
+// ---------------------------------------------------------------------------
+
+export async function rpDetect(
+  imageB64: string,
+  width: number,
+  height: number,
+): Promise<RunpodDetection[]> {
+  const output = await runpodCall<unknown>({
+    name: 'yolo-detect',
+    url: endpointUrl('RUNPOD_YOLO_URL'),
+    input: { image: imageB64, task: 'detect' },
+  });
+  return normalizeDetections(output, width, height);
+}
+
+function extractDetectionArray(output: unknown): unknown[] {
+  if (Array.isArray(output)) return output;
+  if (output && typeof output === 'object') {
+    const o = output as Record<string, unknown>;
+    for (const key of ['detections', 'predictions', 'objects', 'results', 'boxes']) {
+      if (Array.isArray(o[key])) return o[key] as unknown[];
+    }
+  }
+  return [];
+}
+
+/** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */
+function firstLabel(...vals: unknown[]): string | undefined {
+  for (const v of vals) {
+    if (typeof v === 'string' && v.trim()) return v.trim();
+  }
+  return undefined;
+}
+
+function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] {
+  const out: RunpodDetection[] = [];
+  for (const raw of extractDetectionArray(output)) {
+    if (!raw || typeof raw !== 'object') continue;
+    const o = raw as Record<string, unknown>;
+    // Prefer a human-readable name (ultralytics tojson puts the string in `name`
+    // and a numeric index in `class`); fall back to class_<id>. Using firstLabel
+    // (not `??`) also means an explicit empty-string label doesn't get kept + dropped.
+    const classId = o.class_id ?? (typeof o.class === 'number' ? o.class : undefined);
+    const label = (
+      firstLabel(o.label, o.name, o.class_name, typeof o.class === 'string' ? o.class : undefined) ??
+      (classId != null ? `class_${classId}` : 'object')
+    ).toLowerCase();
+    const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0;
+    const box = normalizeBox(o, width, height);
+    if (box && label) out.push({ label, confidence, box });
+  }
+  return out;
+}
+
+function num(v: unknown): number | null {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+}
+
+function asNum4(v: unknown): [number, number, number, number] | null {
+  if (!Array.isArray(v) || v.length < 4) return null;
+  const a = num(v[0]);
+  const b = num(v[1]);
+  const c = num(v[2]);
+  const d = num(v[3]);
+  return a === null || b === null || c === null || d === null ? null : [a, b, c, d];
+}
+
+/**
+ * Normalize a detection box to {x, y, width, height} as fractions 0..1 of the
+ * image, from whatever shape the endpoint emits:
+ *   - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form
+ *   - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form
+ *   - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners
+ *   - object `{x,y,width,height}` → x/y/width/height
+ * Pixel values (any component > 1) are divided by the image dims; already-
+ * normalized fractions pass through. VERIFY the shape against your live #1 endpoint.
+ */
+function normalizeBox(o: Record<string, unknown>, width: number, height: number): RunpodDetection['box'] | null {
+  const W = width || 1;
+  const H = height || 1;
+  const clamp01 = (n: number) => Math.max(0, Math.min(1, n));
+  const frac = (x: number, y: number, w: number, h: number): RunpodDetection['box'] => {
+    if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) {
+      x /= W;
+      y /= H;
+      w /= W;
+      h /= H;
+    }
+    return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) };
+  };
+  const fromXyxy = (x1: number, y1: number, x2: number, y2: number) => frac(x1, y1, x2 - x1, y2 - y1);
+
+  // 1. Array boxes, interpreted by which key holds them.
+  const xyxyArr = asNum4(o.xyxy);
+  if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]);
+  const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h]
+  if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]);
+  const boxArr = asNum4(o.box); // generic array box → assume corner form
+  if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
+
+  // 2. Object boxes (either nested under `box` or directly on the detection).
+  const src = o.box && typeof o.box === 'object' && !Array.isArray(o.box) ? (o.box as Record<string, unknown>) : o;
+  const x1 = num(src.x1 ?? src.left);
+  const y1 = num(src.y1 ?? src.top);
+  const x2 = num(src.x2 ?? src.right);
+  const y2 = num(src.y2 ?? src.bottom);
+  if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2);
+
+  const x = num(src.x);
+  const y = num(src.y);
+  const w = num(src.width);
+  const h = num(src.height);
+  if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h);
+
+  return null;
+}
+
+// ---------------------------------------------------------------------------
+// Scaffold-only endpoints (typed + wired to env, but no route/UI yet).
+// See docs/AI-SETUP.md → "Not yet wired". Kept so the client covers all 15
+// endpoints and a future feature can call them without new plumbing.
+// ---------------------------------------------------------------------------
+
+/** #2 Camera tilt (DeepSingleImageCalibration). */
+export async function rpTilt(imageB64: string): Promise<TiltResult> {
+  const o = await runpodCall<Record<string, unknown>>({
+    name: 'tilt',
+    url: endpointUrl('RUNPOD_TILT_URL'),
+    input: { image: imageB64 },
+  });
+  return {
+    rollDegrees: Number(o.roll_degrees ?? 0) || 0,
+    pitchDegrees: Number(o.pitch_degrees ?? 0) || 0,
+    fovDegrees: Number(o.fov_degrees ?? 0) || 0,
+  };
+}
+
+/** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */
+export async function rpTranscribe(
+  audioB64: string,
+  opts?: { language?: string; timestamps?: boolean; punctuation?: boolean },
+): Promise<TranscriptResult> {
+  const o = await runpodCall<Record<string, unknown>>({
+    name: 'transcribe',
+    url: endpointUrl('RUNPOD_STT_URL'),
+    input: { audio: audioB64, task: 'transcribe', ...(opts ?? {}) },
+  });
+  const rawSegments = Array.isArray(o.segments) ? o.segments : [];
+  const segments = rawSegments.map((s) => {
+    const seg = (s ?? {}) as Record<string, unknown>;
+    return {
+      text: String(seg.text ?? ''),
+      startSec: Number(seg.start_sec ?? 0) || 0,
+      endSec: Number(seg.end_sec ?? 0) || 0,
+    };
+  });
+  return { transcript: String(o.transcript ?? ''), segments: segments.length ? segments : undefined };
+}
+
+/** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */
+export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> {
+  const o = await runpodCall<Record<string, unknown>>({
+    name: 'audio-denoise',
+    url: endpointUrl('RUNPOD_AUDIO_DENOISE_URL'),
+    input: { audio: audioB64, task: 'denoise' },
+  });
+  const a = o.audio ?? o.output ?? '';
+  return { audioB64: stripDataUri(String(a)) };
+}

apps/web/src/lib/runpod/types.ts

@@ -0,0 +1,62 @@
+/**
+ * Request/response types for the RunPod endpoints described in
+ * `model-api-spec.docx`. Server-side only — imported by the /api/ai/* routes,
+ * never by client/SDK code (which must not see a RunPod URL or key).
+ */
+
+/** Normalized image result returned by every image endpoint (raw base64, no data: prefix). */
+export interface RunpodImageResult {
+  imageBase64: string;
+  mimeType: string;
+}
+
+/** #9 SD 3.5 masked inpainting (also #11 outpaint, pre-padded). white in mask = regenerate. */
+export interface InpaintReq {
+  imageB64: string;
+  maskB64: string;
+  prompt: string;
+  negativePrompt?: string;
+  strength?: number;
+  guidanceScale?: number;
+  steps?: number;
+  seed?: number;
+}
+
+/** #10 SD 3.5 general prompt edit (img2img, no mask). */
+export interface Img2ImgReq {
+  imageB64: string;
+  prompt: string;
+  negativePrompt?: string;
+  strength?: number;
+  guidanceScale?: number;
+  steps?: number;
+  seed?: number;
+}
+
+/**
+ * #1 YOLO construction-material classifier, normalized to the SDK's
+ * `DetectedObject` shape (box as fractions 0..1 of the image).
+ */
+export interface RunpodDetection {
+  label: string;
+  confidence: number;
+  box: { x: number; y: number; width: number; height: number };
+}
+
+/** #2 DeepSingleImageCalibration — camera tilt (scaffold; no route/UI yet). */
+export interface TiltResult {
+  rollDegrees: number;
+  pitchDegrees: number;
+  fovDegrees: number;
+}
+
+/** #3 Parakeet voice-to-text (scaffold; no route/UI yet). */
+export interface TranscriptSegment {
+  text: string;
+  startSec: number;
+  endSec: number;
+}
+export interface TranscriptResult {
+  transcript: string;
+  segments?: TranscriptSegment[];
+}

apps/web/src/lib/seed.ts

@@ -137,5 +137,8 @@ export function buildSeedPhotos(now = Date.now()): MediaInput[] {
     });
   }
 
-  return items;
+  // TESTING: seed only the demo video(s) — start with an EMPTY photo library so you
+  // can upload 1–2 images and watch them get analyzed. To restore the full demo,
+  // change this back to `return items;`.
+  return items.filter((it) => it.kind === 'video');
 }

apps/web/src/middleware.ts

@@ -10,7 +10,6 @@ import { type NextRequest, NextResponse } from 'next/server';
  */
 export function middleware(request: NextRequest) {
   const nonce = Buffer.from(crypto.randomUUID()).toString('base64');
-  const isDev = process.env.NODE_ENV === 'development';
 
   const csp = [
     "default-src 'self'",
@@ -18,7 +17,10 @@ export function middleware(request: NextRequest) {
     // cdn.jsdelivr.net = defense for tesseract.js' worker bootstrap (the load-bearing
     // path is the blob: worker + connect-src fetch; under 'strict-dynamic' this host is
     // ignored, but it covers non-strict-dynamic fallback browsers).
-    `script-src 'self' 'nonce-${nonce}' 'strict-dynamic' 'wasm-unsafe-eval' https://cdn.jsdelivr.net${isDev ? " 'unsafe-eval'" : ''}`,
+    // 'unsafe-eval' is required by onnxruntime-web (Remove Background @imgly, CLIP
+    // semantic search) which evals a JS glue string — 'wasm-unsafe-eval' alone
+    // doesn't cover it. Trade-off: needed for the in-browser ML to run in production.
+    `script-src 'self' 'nonce-${nonce}' 'strict-dynamic' 'wasm-unsafe-eval' 'unsafe-eval' https://cdn.jsdelivr.net`,
     // Leaflet & Framer Motion inject inline styles; keep style inline allowed.
     "style-src 'self' 'unsafe-inline'",
     // *.supabase.co serves uploaded media from Supabase Storage public URLs.
@@ -30,7 +32,7 @@ export function middleware(request: NextRequest) {
     // cdn.jsdelivr.net = face-api models + tesseract.js worker/WASM-core/traineddata (OCR) + onnxruntime WASM;
     // huggingface.co + *.hf.co = transformers.js CLIP model config + weights, which
     // redirect to HF's Xet CDN (e.g. us.aws.cdn.hf.co) — semantic search.
-    "connect-src 'self' blob: data: https://*.tile.openstreetmap.org https://server.arcgisonline.com https://picsum.photos https://fastly.picsum.photos https://storage.googleapis.com https://*.supabase.co https://staticimgly.com https://cdn.jsdelivr.net https://huggingface.co https://*.huggingface.co https://*.hf.co",
+    "connect-src 'self' blob: data: https://*.tile.openstreetmap.org https://nominatim.openstreetmap.org https://server.arcgisonline.com https://picsum.photos https://fastly.picsum.photos https://storage.googleapis.com https://*.supabase.co https://staticimgly.com https://cdn.jsdelivr.net https://huggingface.co https://*.huggingface.co https://*.hf.co",
     "worker-src 'self' blob:",
     "frame-ancestors 'none'",
     "base-uri 'self'",