diff --git a/.env.local.example b/.env.local.example index e002499..6295e28 100644 --- a/.env.local.example +++ b/.env.local.example @@ -14,3 +14,83 @@ BFF_ORIGIN=http://localhost:4000 # Installing @abe-kap/appshell-sdk (GitHub Packages) needs a read:packages token: # locally: export NODE_AUTH_TOKEN= before npm install # Vercel: set NODE_AUTH_TOKEN as a project env var + + +# =========================================================================== +# SMART GALLERY — AI routes (/api/gallery/ai/*). Full docs: docs/SMART_GALLERY.md +# =========================================================================== +# +# AUTH: these routes are session-gated. When NEXT_PUBLIC_SUPABASE_URL above is +# SET, every AI request is verified against ${BFF_ORIGIN}/api/session/context and +# a non-200 is rejected. When it is UNSET the app is in local demo mode and the +# AI routes are UNAUTHENTICATED — never expose such a deployment publicly while +# RUNPOD_API_KEY is set, or anyone can spend your GPU budget. +# +# NOTHING below is required to run the gallery: object detection, faces, OCR and +# semantic search all run FREE in-browser with no key. Only generative editing, +# transcription, denoise and tilt need RunPod. + +# --- RunPod credentials ----------------------------------------------------- +# Server-side ONLY. Never prefix with NEXT_PUBLIC_ — that would ship the key to +# the browser. Read exclusively by src/lib/server/runpod/client.ts. +RUNPOD_API_KEY=rpa_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# --- Per-model endpoint URLs (deploy each so the URL ends in /runsync) ------- +# Several models can share one endpoint id — that is expected, not a mistake. +RUNPOD_SD_IMG2IMG_URL=https://api.runpod.ai/v2//runsync # #10 prompt / colorize (img2img) +RUNPOD_SD_INPAINT_URL=https://api.runpod.ai/v2//runsync # #9 replace-sky / magic-eraser / generative-fill / outpaint (masked) +RUNPOD_YOLO_URL=https://api.runpod.ai/v2//runsync # #1 object detection → /api/gallery/ai/classify +RUNPOD_UPSCALE_URL=https://api.runpod.ai/v2//runsync # #7 restore / upscale (Real-ESRGAN) +RUNPOD_STT_URL=https://api.runpod.ai/v2//runsync # #3 speech-to-text → /api/gallery/ai/transcribe +RUNPOD_BG_REMOVE_URL=https://api.runpod.ai/v2//runsync # #6 background removal (U²-Net) +RUNPOD_AUDIO_DENOISE_URL=https://api.runpod.ai/v2//runsync # #12 audio denoise → /api/gallery/ai/denoise +# Optional, only if you deploy them: +# RUNPOD_TILT_URL=https://api.runpod.ai/v2//runsync # #2 camera tilt → /api/gallery/ai/tilt (needs the switch below) +# RUNPOD_COLORIZE_URL=https://api.runpod.ai/v2//runsync # #8 DDColor (currently unused — colorize goes through img2img) + +# --- Which backend serves /api/gallery/ai/edit ------------------------------ +# auto = first configured of: runpod → local → huggingface → gemini +# runpod = the RUNPOD_*_URL endpoints above (recommended) +# local = your own Stable Diffusion (A1111/Forge/SD.Next); needs LOCAL_SD_URL +# huggingface = HF Inference API; needs HF_API_TOKEN (+ optional HF_IMAGE_MODEL) +# gemini = Google Gemini; needs GEMINI_API_KEY (image output requires a BILLED key) +# none = disable generative editing entirely (503 with a clear message) +AI_EDIT_PROVIDER=runpod + +# Alternative backends (only read when AI_EDIT_PROVIDER selects them): +# LOCAL_SD_URL=http://127.0.0.1:7860 +# LOCAL_SD_DENOISE=0.55 +# LOCAL_SD_STEPS=25 +# LOCAL_SD_SAMPLER=Euler a +# HF_API_TOKEN=hf_xxxxxxxx +# HF_IMAGE_MODEL=timbrooks/instruct-pix2pix +# GEMINI_API_KEY= +# GEMINI_IMAGE_MODEL=gemini-2.5-flash-image + +# Optional Stable Diffusion tuning (defaults in src/lib/server/runpod/endpoints.ts): +# RUNPOD_SD_STEPS=35 +# RUNPOD_SD_STRENGTH=0.8 +# RUNPOD_SD_GUIDANCE=7 +# RUNPOD_SD_NEGATIVE_PROMPT= + +# --- Client-side capability switches (NEXT_PUBLIC_, inlined at BUILD time) --- +# Each must be the literal string "true" to enable; anything else is off. Because +# they are inlined at build time, changing one requires a rebuild, not a restart. +# +# false/unset = object detection runs FREE in-browser (COCO-SSD). true = use the +# RunPod YOLO classifier via /api/gallery/ai/classify (COCO-SSD stays the fallback). +# NEXT_PUBLIC_APG_RUNPOD_DETECT=false +# +# false/unset = background removal runs in-browser (@imgly WASM, no key). +# true = use the RunPod U²-Net endpoint, falling back to @imgly on failure. +NEXT_PUBLIC_APG_RUNPOD_BG=true +# +# true = show the editor's Auto-straighten button and call /api/gallery/ai/tilt. +# Only enable this if RUNPOD_TILT_URL is actually deployed. +# NEXT_PUBLIC_APG_RUNPOD_TILT=false + +# NOTE: the photo-gallery SDK also reads a family of NEXT_PUBLIC_APG_* THEMING +# vars (NEXT_PUBLIC_APG_THEME / _ACCENT / _RADIUS / _BG_DARK / _SIDEBAR_BG_* …) +# in its standalone demo. Those are NOT used here — the CRM passes `themeTokens` +# to the gallery component directly so the gallery inherits the dashboard's +# design tokens. Setting them in this file has no effect. diff --git a/docs/SMART_GALLERY.md b/docs/SMART_GALLERY.md new file mode 100644 index 0000000..b68b8fd --- /dev/null +++ b/docs/SMART_GALLERY.md @@ -0,0 +1,230 @@ +# Smart Gallery — AI route surface + +The Smart Gallery embeds `@photo-gallery/sdk` into the CRM. The SDK never talks to a +model directly: it calls a pluggable `AIProvider`, which the CRM supplies via +`createCrmAIProvider()` in [`src/lib/gallery-ai.ts`](../src/lib/gallery-ai.ts). + +Roughly half the intelligence runs **in the browser** for free, and the other half is +proxied through **five server routes** under `/api/gallery/ai/*` so the RunPod API key +never reaches the client. + +--- + +## 1. Where each capability runs + +| Capability | Where | Backend | Needs a key? | +| --- | --- | --- | --- | +| Object detection (default) | Browser | TensorFlow.js COCO-SSD (80 COCO classes) | No | +| Object detection (opt-in) | **Server** | RunPod YOLO → `/classify` | Yes | +| Face detection + recognition | Browser | `@vladmandic/face-api` (128-D descriptors → People) | No | +| OCR / document search | Browser | `tesseract.js` | No | +| Semantic ("beach photos") search | Browser | CLIP via `@huggingface/transformers` | No | +| Background removal (default) | Browser | `@imgly/background-removal` (WASM) | No | +| Background removal (opt-in) | **Server** | RunPod U²-Net → `/edit` | Yes | +| Generative edits, restore, upscale, outpaint | **Server** | `/edit` | Yes | +| Speech-to-text | **Server** | `/transcribe` | Yes | +| Audio denoise | **Server** | `/denoise` | Yes | +| Camera tilt / auto-straighten | **Server** | `/tilt` | Yes | + +Every in-browser model is loaded with `await import(...)` on first use, so none of it +lands in the initial bundle. **Every capability degrades gracefully** — a model that +fails to load returns `[]` / `''` / `null` with a `console.warn` rather than throwing, +so a blocked CDN never breaks the gallery UI. + +--- + +## 2. Auth model + +All five routes share one gate: `requireGallerySession()` in +[`src/lib/server/session.ts`](../src/lib/server/session.ts). + +These routes proxy a **paid, rate-limited GPU backend** using a secret held only on the +server. An unauthenticated route here is not just an information leak — it is an open +invitation to spend the operator's GPU budget. (The upstream SDK demo's routes are +completely unauthenticated; that is the single biggest thing this port fixes.) + +**When the Shell is configured** (`NEXT_PUBLIC_SUPABASE_URL` is set): the incoming +`cookie` header is forwarded to `${BFF_ORIGIN}/api/session/context` and only a `200` +is accepted. + +- `401` upstream → `401 { error: "Not signed in" }` +- any other non-200, or a network/timeout failure → `503 { error: "Session service unavailable" }` + (**fail closed** — if we cannot prove a session, we do not spend GPU budget) +- **Nothing is cached.** A cached "yes" would keep a revoked session alive, so every AI + request costs one BFF round trip. Correctness over latency for a spend gate. +- The BFF is trusted absolutely — `BFF_ORIGIN` must only ever point at an origin the + operator controls. + +**When the Shell is NOT configured** (local demo mode): the request is allowed and a +warning is logged once. **Never deploy to a public origin with the Shell unconfigured +and a real `RUNPOD_API_KEY` present** — that combination is an open, billable endpoint. + +This is authentication only, not authorization. Per-resource gallery policy lives in +be-crm behind the `crm.gallery` resource. + +--- + +## 3. Rate limits + +[`src/lib/server/rate-limit.ts`](../src/lib/server/rate-limit.ts) — a fixed-window +counter keyed by the authenticated principal when known, otherwise the first hop of +`x-forwarded-for`. Each route has its own namespace, so spending your `edit` budget +does not consume your `classify` budget. Exceeding it returns +`429 { error: "Too many requests — slow down." }` with a `Retry-After` header. + +| Route | Limit | +| --- | --- | +| `/classify` | 30 / min | +| `/tilt` | 30 / min | +| `/transcribe` | 20 / min | +| `/denoise` | 20 / min | +| `/edit` | 12 / min (most expensive) | + +> **This limiter is per-instance and in-memory.** With N instances behind a load +> balancer a caller gets up to N x the budget, and a restart clears all counters. It is +> a cost guard, not a security boundary. **Replace it with Redis before running more +> than one instance.** The `x-forwarded-for` key is also client-controlled unless a +> trusted proxy overwrites it — the session gate, not this, is the security boundary. + +--- + +## 4. The routes + +All five are `POST` only, and all declare `runtime = "nodejs"`, `maxDuration = 60`, +`dynamic = "force-dynamic"`. Failures always return `{ error: string }`. + +Shared status codes: `400` invalid body/params · `401` not signed in · `413` payload too +large · `429` rate limited · `500` server misconfigured (e.g. missing `RUNPOD_API_KEY`) · +`502` upstream failed · `503` not configured / session service unavailable · +`504` upstream timed out (the RunPod client's budget is 55s, under the 60s cap). + +Upstream error bodies are truncated to a **160-character excerpt**; the RunPod key is +never included in any response. + +### `POST /api/gallery/ai/classify` — object detection +Backed by `RUNPOD_YOLO_URL`. Max ~4 MB of base64. Returns boxes as **fractions 0..1** +of the image, matching the SDK's `DetectedObject`. + +```jsonc +// request +{ "imageBase64": "…", "width": 1280, "height": 853 } +// response +{ "objects": [ { "label": "excavator", "confidence": 0.91, + "box": { "x": 0.12, "y": 0.30, "width": 0.25, "height": 0.40 } } ] } +``` + +### `POST /api/gallery/ai/edit` — generative image editing +The backend is pluggable via `AI_EDIT_PROVIDER` (`auto` | `runpod` | `local` | +`huggingface` | `gemini` | `none`). Image + mask are budgeted **together** against the +~4 MB cap. Prompts come from an allow-listed op set — arbitrary server-side prompts are +never accepted, and free-text is clamped to 500 chars. + +```jsonc +// request +{ "imageBase64": "…", "mimeType": "image/jpeg", + "op": { "type": "restore" }, // or prompt | colorize | replace-sky | + // magic-eraser | generative-fill | upscale | + // remove-background + "maskBase64": "…", // required for magic-eraser / generative-fill + "params": { "strength": 0.8 } } +// response +{ "imageBase64": "…", "mimeType": "image/png" } +``` + +Op → RunPod endpoint: `restore`/`upscale` → `RUNPOD_UPSCALE_URL` · +`prompt`/`colorize` → `RUNPOD_SD_IMG2IMG_URL` · `replace-sky`/`magic-eraser`/ +`generative-fill` → `RUNPOD_SD_INPAINT_URL` · `remove-background` → +`RUNPOD_BG_REMOVE_URL`. `upscale`, `magic-eraser` and `generative-fill` are RunPod-only +and return `400` under another backend. Outpaint is client-side padding plus a +`generative-fill` call — it needs no separate route. + +### `POST /api/gallery/ai/tilt` — camera tilt +Backed by `RUNPOD_TILT_URL`. Max ~4 MB (`413` over). Only reachable when +`NEXT_PUBLIC_APG_RUNPOD_TILT=true`, which also reveals the editor's Auto-straighten button. + +```jsonc +{ "image": "…" } → { "rollDegrees": -2.4, "pitchDegrees": 1.1, "fovDegrees": 68.2 } +``` + +### `POST /api/gallery/ai/transcribe` — speech to text +Backed by `RUNPOD_STT_URL`. Expects base64 **WAV 16 kHz mono PCM16**. Max ~8 MB (`413`). + +```jsonc +{ "audio": "…", "language": "en" } +→ { "transcript": "…", "segments": [ { "text": "…", "startSec": 0, "endSec": 1.8 } ] } +``` + +### `POST /api/gallery/ai/denoise` — audio noise removal +Backed by `RUNPOD_AUDIO_DENOISE_URL`. Expects base64 **WAV 48 kHz mono PCM16**. Max +~12 MB (`413`). Used before transcription on noisy sites. + +```jsonc +{ "audio": "…" } → { "audio": "…" } +``` + +--- + +## 5. Environment variables + +See [`.env.local.example`](../.env.local.example) for the fully commented template. + +| Var | Backs | +| --- | --- | +| `RUNPOD_API_KEY` | all five routes (server-only — never `NEXT_PUBLIC_`) | +| `RUNPOD_YOLO_URL` | `/classify` | +| `RUNPOD_SD_IMG2IMG_URL` | `/edit` — prompt, colorize, maskless replace-sky | +| `RUNPOD_SD_INPAINT_URL` | `/edit` — magic-eraser, generative-fill, outpaint, masked replace-sky | +| `RUNPOD_UPSCALE_URL` | `/edit` — restore, upscale | +| `RUNPOD_BG_REMOVE_URL` | `/edit` — remove-background | +| `RUNPOD_STT_URL` | `/transcribe` | +| `RUNPOD_AUDIO_DENOISE_URL` | `/denoise` | +| `RUNPOD_TILT_URL` | `/tilt` | +| `AI_EDIT_PROVIDER` | which backend `/edit` uses | +| `BFF_ORIGIN` | the session gate | + +Client switches (`NEXT_PUBLIC_`, **inlined at build time** — a change needs a rebuild, +and each must be the literal string `"true"`): +`NEXT_PUBLIC_APG_RUNPOD_DETECT`, `NEXT_PUBLIC_APG_RUNPOD_BG`, `NEXT_PUBLIC_APG_RUNPOD_TILT`. + +> The SDK's standalone demo also reads a family of `NEXT_PUBLIC_APG_*` **theming** vars +> (`_THEME`, `_ACCENT`, `_RADIUS`, `_BG_DARK`, `_SIDEBAR_BG_*`, …). Those are **not used +> here** — the CRM passes `themeTokens` to the gallery directly so it inherits the +> dashboard's design tokens. Setting them in this app has no effect. + +--- + +## 6. CSP / CDN hosts + +**The CRM ships no Content-Security-Policy today**, so the in-browser models load +without any allow-listing. This section is what would need to be permitted **if a CSP +is ever added** — miss any of these and the affected capability silently degrades +(returns empty + `console.warn`) rather than erroring visibly, which makes it easy to +misdiagnose. + +| Host | Needed by | Directive | +| --- | --- | --- | +| `https://cdn.jsdelivr.net` | face-api models; tesseract worker, WASM core **and** language data | `script-src`, `connect-src`, `worker-src` | +| `https://huggingface.co`, `https://cdn-lfs.huggingface.co` | CLIP model weights (transformers.js) | `connect-src` | +| `https://storage.googleapis.com` | TensorFlow.js COCO-SSD model | `connect-src` | +| `https://staticimgly.com` | `@imgly/background-removal` WASM + assets | `connect-src`, `worker-src` | + +Also required by the ML runtimes themselves: + +- **`wasm-unsafe-eval` in `script-src`** — ONNX Runtime (CLIP), tesseract-core and the + `@imgly` remover are all WebAssembly. Without it, semantic search, OCR and in-browser + background removal all fail. +- **`blob:` in `worker-src`/`child_src`** — tesseract and the WASM runtimes spawn + workers from blob URLs. +- **`data:` and `blob:` in `img-src`** — canvas round-trips and generated results. + +Notes: +- All three tesseract assets (worker, core, tessdata) are pinned to **jsDelivr** on + purpose. Left at its defaults, tesseract fetches language data from + `tessdata.projectnaptha.com`, which would be a second host to allow-list. +- `tesseract.js` is pinned to **exactly `5.1.1`** (no caret) in `package.json` because + the worker CDN URL embeds the version — a floating range would let the worker drift + out of sync with the installed main-thread code. +- The tfjs providers prefer the **WebGL** backend, which avoids `eval` and is therefore + CSP-friendly; they fall back to the default backend if WebGL is unavailable. +- The SDK's own demo app runs a strict per-request nonce CSP with these hosts already + allow-listed — see that repo's middleware for a working reference. diff --git a/next.config.ts b/next.config.ts index aa1c5aa..640a5b4 100644 --- a/next.config.ts +++ b/next.config.ts @@ -1,8 +1,9 @@ import type { NextConfig } from "next"; const nextConfig: NextConfig = { - // The SDK ships ESM/TS; let Next transpile it. - transpilePackages: ["@abe-kap/appshell-sdk"], + // Both SDKs ship ESM/TS source (their package `exports` point at src/), so Next + // must transpile them rather than treat them as prebuilt CJS. + transpilePackages: ["@abe-kap/appshell-sdk", "@photo-gallery/sdk"], // The browser calls the Shell BFF same-origin under /shell (so the HttpOnly // session cookie flows). We deliberately use /shell (NOT /api) to avoid // clobbering the existing /api/geo route. Point BFF_ORIGIN at the deployed BFF. diff --git a/package.json b/package.json index 726f655..fe4017b 100644 --- a/package.json +++ b/package.json @@ -6,20 +6,30 @@ "dev": "next dev", "build": "next build", "start": "next start", - "lint": "eslint" + "lint": "eslint", + "sync:gallery-sdk": "node scripts/sync-gallery-sdk.mjs" }, "dependencies": { "@abe-kap/appshell-sdk": "^0.2.6", + "@huggingface/transformers": "^4.2.0", + "@imgly/background-removal": "^1.7.0", "@insignia/iios-kernel-client": "^0.1.4", + "@photo-gallery/sdk": "file:./vendor/photo-gallery-sdk", + "@tensorflow-models/coco-ssd": "^2.2.3", + "@tensorflow/tfjs": "^4.22.0", + "@vladmandic/face-api": "^1.7.15", "clsx": "^2.1.1", + "leaflet": "^1.9.4", "lucide-react": "^1.21.0", "next": "16.2.9", "react": "19.2.4", "react-dom": "19.2.4", - "tailwind-merge": "^3.6.0" + "tailwind-merge": "^3.6.0", + "tesseract.js": "5.1.1" }, "devDependencies": { "@tailwindcss/postcss": "^4", + "@types/leaflet": "^1.9.21", "@types/node": "^20", "@types/react": "^19", "@types/react-dom": "^19", diff --git a/scripts/sync-gallery-sdk.mjs b/scripts/sync-gallery-sdk.mjs new file mode 100644 index 0000000..307aa9c --- /dev/null +++ b/scripts/sync-gallery-sdk.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +/** + * Re-sync the vendored @photo-gallery/sdk from the sibling SDK checkout. + * + * The SDK is not published to a registry yet, so it is vendored into `vendor/photo-gallery-sdk` + * and depended on as `file:./vendor/photo-gallery-sdk` — a path INSIDE this repo, so CI and Vercel + * (which only ever check out this repo) can resolve it. See vendor/README.md. + * + * npm run sync:gallery-sdk + * git add vendor/photo-gallery-sdk && git commit -m "chore: sync @photo-gallery/sdk" + * + * Only what the package would publish is copied. Its `node_modules` is deliberately left behind: + * without it the SDK's sources resolve `@types/react` from this repo (React 19) instead of the + * pnpm workspace's React 18, which is what lets them type-check here. + */ + +import { cp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const source = + process.env.GALLERY_SDK_PATH ?? + resolve(root, "../advance-photo-gallery-web-sdk/packages/photo-sdk"); +const target = resolve(root, "vendor/photo-gallery-sdk"); + +const ENTRIES = ["src", "package.json", "README.md"]; + +const exists = async (p) => { + try { + await stat(p); + return true; + } catch { + return false; + } +}; + +if (!(await exists(source))) { + console.error(`[sync:gallery-sdk] SDK checkout not found at ${source}`); + console.error(" Clone advance-photo-gallery-web-sdk beside this repo, or set GALLERY_SDK_PATH."); + process.exit(1); +} + +for (const entry of ENTRIES) { + const from = resolve(source, entry); + if (!(await exists(from))) continue; + const to = resolve(target, entry); + await rm(to, { recursive: true, force: true }); + await cp(from, to, { recursive: true }); + console.log(`[sync:gallery-sdk] ${entry}`); +} + +/* + * Strip devDependencies + scripts from the vendored manifest. + * + * npm never installs a published package's devDependencies, but it DOES install them for a local + * `file:` path package. Left in place, the SDK's `@types/react@18` lands in + * vendor/photo-gallery-sdk/node_modules and its sources then type-check against React 18 inside a + * React 19 program — "Type 'bigint' is not assignable to type 'ReactNode'". Removing them makes the + * vendored copy behave exactly like the tarball it stands in for. + */ +const manifestPath = resolve(target, "package.json"); +const manifest = JSON.parse(await readFile(manifestPath, "utf8")); +delete manifest.devDependencies; +delete manifest.scripts; +manifest._vendored = { + from: "advance-photo-gallery-web-sdk/packages/photo-sdk", + note: "Generated by scripts/sync-gallery-sdk.mjs — do not hand-edit. See vendor/README.md.", +}; +await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); +console.log("[sync:gallery-sdk] package.json (devDependencies + scripts stripped)"); + +console.log("[sync:gallery-sdk] done — commit vendor/photo-gallery-sdk, then restart the dev server."); diff --git a/src/app/api/gallery/ai/_guard.ts b/src/app/api/gallery/ai/_guard.ts new file mode 100644 index 0000000..2a88ebf --- /dev/null +++ b/src/app/api/gallery/ai/_guard.ts @@ -0,0 +1,61 @@ +/** + * Shared entry gate for every /api/gallery/ai/* route: authenticate, then + * throttle. Lives in a `_`-prefixed file so the App Router never treats it as a + * route (only `route.ts` defines an endpoint). + * + * Order matters: we authenticate FIRST so the rate limit can be keyed by + * principal rather than by a spoofable `x-forwarded-for` hop wherever possible. + * The session check is one cheap BFF round trip; the work it guards is a GPU + * call, so paying it before throttling is the right trade. + * + * SERVER-ONLY. + */ + +import { NextResponse } from "next/server"; + +import { limit } from "@/lib/server/rate-limit"; +import { rateLimitKey, requireGallerySession } from "@/lib/server/session"; + +/** Per-minute budgets, per the Smart Gallery route contract. */ +export const RATE_LIMITS = { + classify: 30, + edit: 12, + tilt: 30, + transcribe: 20, + denoise: 20, +} as const; + +const WINDOW_MS = 60_000; + +export type GuardResult = + | { ok: true; principalId?: string } + /** Ready-to-return error response — the route should return it unchanged. */ + | { ok: false; response: NextResponse }; + +/** + * @param route Which budget to apply (also namespaces the limiter key so a + * caller's `edit` spend does not consume their `classify` budget). + */ +export async function guard(req: Request, route: keyof typeof RATE_LIMITS): Promise { + const session = await requireGallerySession(req); + if (!session.ok) { + return { + ok: false, + response: NextResponse.json({ error: session.error }, { status: session.status }), + }; + } + + const key = `${route}:${rateLimitKey(req, session.principalId)}`; + const { ok, retryAfter } = limit(key, RATE_LIMITS[route], WINDOW_MS); + if (!ok) { + return { + ok: false, + response: NextResponse.json( + { error: "Too many requests — slow down." }, + { status: 429, headers: { "Retry-After": String(retryAfter) } }, + ), + }; + } + + return { ok: true, principalId: session.principalId }; +} diff --git a/src/app/api/gallery/ai/classify/route.ts b/src/app/api/gallery/ai/classify/route.ts new file mode 100644 index 0000000..3aeb216 --- /dev/null +++ b/src/app/api/gallery/ai/classify/route.ts @@ -0,0 +1,71 @@ +import { type NextRequest, NextResponse } from "next/server"; + +import { RunpodError } from "@/lib/server/runpod/client"; +import { rpDetect } from "@/lib/server/runpod/endpoints"; + +import { guard } from "../_guard"; + +export const runtime = "nodejs"; +export const maxDuration = 60; +export const dynamic = "force-dynamic"; + +/** + * Object-detection proxy for the RunPod YOLO construction-material classifier (#1). + * The key + endpoint URL stay server-side. The client calls this only when + * NEXT_PUBLIC_APG_RUNPOD_DETECT is on; otherwise detection runs fully in-browser + * (COCO-SSD) with no server round-trip. Returns the SDK's DetectedObject[] shape + * (box as 0..1 fractions) so it drops straight into the Objects browser / smart + * albums / search. + * + * POST { imageBase64, width, height } -> { objects: [{ label, confidence, box }] } + * Auth: session-gated (see lib/server/session.ts). Rate limit: 30/min. + */ + +const MAX_BASE64 = 4_000_000; // ~3 MB decoded — under serverless body limits + +export async function POST(req: NextRequest) { + const gate = await guard(req, "classify"); + if (!gate.ok) return gate.response; + + if (!process.env.RUNPOD_API_KEY || !process.env.RUNPOD_YOLO_URL) { + return NextResponse.json( + { error: "RunPod detection is not configured (set RUNPOD_API_KEY + RUNPOD_YOLO_URL)." }, + { status: 503 }, + ); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid request body." }, { status: 400 }); + } + + const { imageBase64, width, height } = (body ?? {}) as { + imageBase64?: unknown; + width?: unknown; + height?: unknown; + }; + if ( + typeof imageBase64 !== "string" || + imageBase64.length === 0 || + imageBase64.length > MAX_BASE64 + ) { + return NextResponse.json({ error: "Invalid or oversized image." }, { status: 400 }); + } + const w = Number(width); + const h = Number(height); + + try { + const objects = await rpDetect( + imageBase64, + Number.isFinite(w) && w > 0 ? w : 1, + Number.isFinite(h) && h > 0 ? h : 1, + ); + return NextResponse.json({ objects }); + } catch (err) { + const message = err instanceof Error ? err.message : "Detection failed."; + const status = err instanceof RunpodError ? err.status : 502; + return NextResponse.json({ error: message }, { status }); + } +} diff --git a/src/app/api/gallery/ai/denoise/route.ts b/src/app/api/gallery/ai/denoise/route.ts new file mode 100644 index 0000000..95dc649 --- /dev/null +++ b/src/app/api/gallery/ai/denoise/route.ts @@ -0,0 +1,48 @@ +import { type NextRequest, NextResponse } from "next/server"; + +import { RunpodError } from "@/lib/server/runpod/client"; +import { rpDenoiseAudio } from "@/lib/server/runpod/endpoints"; + +import { guard } from "../_guard"; + +export const runtime = "nodejs"; +export const maxDuration = 60; // cold-start denoise worker can take a while +export const dynamic = "force-dynamic"; + +/** + * Audio noise-removal proxy. Accepts base64 WAV (48 kHz mono PCM16, produced + * in-browser) and returns a cleaned base64 WAV. Calls the RunPod audio-denoise + * endpoint (RUNPOD_AUDIO_DENOISE_URL) — key stays server-side. Used before + * transcription on noisy sites. + * + * POST { audio } -> { audio } + * Auth: session-gated. Rate limit: 20/min. + */ + +const MAX_BASE64 = 12_000_000; // ~9 MB decoded WAV + +export async function POST(req: NextRequest) { + const gate = await guard(req, "denoise"); + if (!gate.ok) return gate.response; + + let body: { audio?: unknown }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 }); + } + + const audio = typeof body.audio === "string" ? body.audio : ""; + if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 }); + if (audio.length > MAX_BASE64) { + return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 }); + } + + try { + const { audioB64 } = await rpDenoiseAudio(audio); + return NextResponse.json({ audio: audioB64 }); + } catch (e) { + const msg = e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Denoise failed."; + return NextResponse.json({ error: msg }, { status: 502 }); + } +} diff --git a/src/app/api/gallery/ai/edit/route.ts b/src/app/api/gallery/ai/edit/route.ts new file mode 100644 index 0000000..2b70aae --- /dev/null +++ b/src/app/api/gallery/ai/edit/route.ts @@ -0,0 +1,419 @@ +import { type NextRequest, NextResponse } from "next/server"; + +import { RunpodError } from "@/lib/server/runpod/client"; +import { + rpImg2Img, + rpInpaint, + rpRemoveBackground, + rpUpscale, +} from "@/lib/server/runpod/endpoints"; + +import { guard } from "../_guard"; + +export const runtime = "nodejs"; +export const maxDuration = 60; // SD / cold-start models can take a while +export const dynamic = "force-dynamic"; + +/** + * Generative image-edit proxy. The BACKEND is pluggable — pick one with env + * `AI_EDIT_PROVIDER` (default `auto`): + * + * - `runpod` → RunPod serverless GPU endpoints (one per model). Maps each + * op → endpoint: restore/upscale → Real-ESRGAN (#7), colorize + * → img2img (#10), replace-sky / magic-eraser / generative-fill + * → SD 3.5 masked inpaint (#9), prompt → SD 3.5 img2img (#10). + * Env: RUNPOD_API_KEY + per-model RUNPOD_*_URL. Key stays + * server-side. + * - `local` → your own Stable Diffusion server (Automatic1111 / Forge / + * SD.Next img2img API). Env: LOCAL_SD_URL. + * - `huggingface` → Hugging Face Inference API. Env: HF_API_TOKEN, HF_IMAGE_MODEL. + * - `gemini` → Google Gemini image model (needs a billed key for image output). + * Env: GEMINI_API_KEY, GEMINI_IMAGE_MODEL. + * - `auto` → first configured of: runpod → local → huggingface → gemini. + * + * NOTE: `remove-background` runs in-browser by default (@imgly, no key), so it + * usually never reaches here. Object detection uses its own route (./classify). + * + * POST { imageBase64, mimeType?, op, maskBase64?, params? } -> { imageBase64, mimeType } + * Auth: session-gated. Rate limit: 12/min (the most expensive route). + */ + +const OP_PROMPTS: Record = { + restore: + "Restore and enhance this photograph: improve sharpness and clarity, correct exposure and white balance, reduce noise and compression artifacts, recover detail. Keep it natural and photorealistic.", + colorize: "Colorize this image with natural, realistic, well-balanced colors.", + "replace-sky": + "Replace the sky with a dramatic, beautiful golden-hour sky with soft clouds. Keep the foreground subject unchanged and the result photorealistic.", +}; + +const MAX_BASE64 = 4_000_000; // ~3 MB decoded — stays under serverless body limits + +type Provider = "runpod" | "local" | "huggingface" | "gemini" | "none"; + +function resolveProvider(): Provider { + const explicit = (process.env.AI_EDIT_PROVIDER || "auto").toLowerCase(); + if ( + explicit === "runpod" || + explicit === "local" || + explicit === "huggingface" || + explicit === "gemini" + ) + return explicit; + if (explicit === "none") return "none"; + // auto: prefer RunPod GPU endpoints, then a private local server, then HF, then Gemini. + // Detect RunPod when the key + ANY image endpoint URL is set (an upscale/colorize-only + // deployment is valid — not just the SD ones). + if ( + process.env.RUNPOD_API_KEY && + (process.env.RUNPOD_SD_IMG2IMG_URL || + process.env.RUNPOD_SD_INPAINT_URL || + process.env.RUNPOD_UPSCALE_URL || + process.env.RUNPOD_COLORIZE_URL || + process.env.RUNPOD_BG_REMOVE_URL) + ) + return "runpod"; + if (process.env.LOCAL_SD_URL) return "local"; + if (process.env.HF_API_TOKEN) return "huggingface"; + if (process.env.GEMINI_API_KEY) return "gemini"; + return "none"; +} + +/** Ops that only the RunPod (mask/fixed-function) backend can serve. */ +const RUNPOD_ONLY_OPS = new Set(["upscale", "magic-eraser", "generative-fill"]); + +interface EditResult { + imageBase64: string; + mimeType: string; +} + +export async function POST(req: NextRequest) { + const gate = await guard(req, "edit"); + if (!gate.ok) return gate.response; + + const provider = resolveProvider(); + if (provider === "none") { + return NextResponse.json( + { + error: + "AI image editing is not configured. Set AI_EDIT_PROVIDER=runpod + RUNPOD_API_KEY + the per-model RUNPOD_*_URL vars (RunPod GPU), or LOCAL_SD_URL (own Stable Diffusion), HF_API_TOKEN (Hugging Face), or GEMINI_API_KEY. Background removal and all analysis still work with no key.", + }, + { status: 503 }, + ); + } + + let body: unknown; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid request body." }, { status: 400 }); + } + + const { imageBase64, mimeType, op, maskBase64, params } = (body ?? {}) as { + imageBase64?: unknown; + mimeType?: unknown; + op?: { type?: string; prompt?: string; factor?: number }; + maskBase64?: unknown; + params?: unknown; + }; + + if (typeof imageBase64 !== "string" || imageBase64.length === 0) { + return NextResponse.json({ error: "Invalid image." }, { status: 400 }); + } + const hasMask = typeof maskBase64 === "string" && maskBase64.length > 0; + // Image + mask share one request body — budget them together against the cap. + if (imageBase64.length + (hasMask ? (maskBase64 as string).length : 0) > MAX_BASE64) { + return NextResponse.json( + { error: "Image (plus mask) is too large — try a smaller image." }, + { status: 400 }, + ); + } + const safeMime = + typeof mimeType === "string" && /^image\/(jpeg|png|webp)$/.test(mimeType) + ? mimeType + : "image/jpeg"; + + const opType = op?.type ?? ""; + if (provider !== "runpod" && RUNPOD_ONLY_OPS.has(opType)) { + return NextResponse.json( + { error: "This edit needs the RunPod backend (set AI_EDIT_PROVIDER=runpod)." }, + { status: 400 }, + ); + } + + // Build the instruction from an allow-listed op (never trust arbitrary server prompts). + let instruction = ""; + if (opType === "prompt" || opType === "generative-fill") { + const p = typeof op?.prompt === "string" ? op.prompt.trim() : ""; + if (!p) return NextResponse.json({ error: "Empty prompt." }, { status: 400 }); + instruction = p.slice(0, 500); + } else if (opType === "replace-sky") { + instruction = + typeof op?.prompt === "string" && op.prompt.trim() + ? `Replace the sky with: ${op.prompt.trim().slice(0, 300)}. Keep the foreground unchanged and photorealistic.` + : OP_PROMPTS["replace-sky"]!; + } else if (opType === "magic-eraser") { + instruction = + "Fill the selected region with a clean, seamless, plausible background. Photorealistic."; + } else if (OP_PROMPTS[opType]) { + instruction = OP_PROMPTS[opType]!; + } else if (opType !== "upscale" && opType !== "remove-background") { + return NextResponse.json({ error: "Unsupported operation." }, { status: 400 }); + } + + try { + let result: EditResult; + if (provider === "runpod") + result = await editRunPod( + op ?? {}, + imageBase64, + instruction, + hasMask ? (maskBase64 as string) : undefined, + params, + ); + else if (provider === "local") result = await editLocal(instruction, imageBase64); + else if (provider === "huggingface") result = await editHuggingFace(instruction, imageBase64); + else result = await editGemini(instruction, imageBase64, safeMime); + return NextResponse.json(result); + } catch (err) { + const message = err instanceof Error ? err.message : "AI request failed."; + const status = err instanceof AiError || err instanceof RunpodError ? err.status : 502; + return NextResponse.json({ error: message }, { status }); + } +} + +// --------------------------------------------------------------------------- +// Backend: RunPod serverless GPU endpoints (one model per endpoint). +// Each op maps to its endpoint; the API key + URLs stay server-side. +// --------------------------------------------------------------------------- +interface SdParams { + negativePrompt?: string; + strength?: number; + steps?: number; + seed?: number; + guidanceScale?: number; +} + +function sanitizeParams(raw: unknown): SdParams { + const p = (raw ?? {}) as Record; + const out: SdParams = {}; + if (typeof p.negativePrompt === "string" && p.negativePrompt.trim()) + out.negativePrompt = p.negativePrompt.trim().slice(0, 300); + const strength = Number(p.strength); + if (Number.isFinite(strength)) out.strength = Math.max(0, Math.min(1, strength)); + const steps = Number(p.steps); + if (Number.isFinite(steps)) out.steps = Math.max(1, Math.min(60, Math.round(steps))); + const guidance = Number(p.guidanceScale); + if (Number.isFinite(guidance)) out.guidanceScale = Math.max(1, Math.min(20, guidance)); + const seed = Number(p.seed); + if (Number.isFinite(seed)) out.seed = Math.max(0, Math.min(2_147_483_647, Math.round(seed))); + return out; +} + +async function editRunPod( + op: { type?: string; prompt?: string; factor?: number }, + imageBase64: string, + instruction: string, + maskBase64: string | undefined, + rawParams: unknown, +): Promise { + const params = sanitizeParams(rawParams); + switch (op.type) { + case "remove-background": + // U²-Net via rembg (#6) — a real endpoint replacing the flaky in-browser remover. + return rpRemoveBackground(imageBase64); + case "restore": + // Real-ESRGAN (#7) with the GFPGAN face pass = "Restore & Enhance". + return rpUpscale(imageBase64, 4, true); + case "upscale": + return rpUpscale(imageBase64, op.factor === 4 ? 4 : 2, false); + case "colorize": + // The dedicated DDColor endpoint kept hard-crashing (modelscope). Route + // colorize through the img2img model as an instruction instead. + return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params }); + case "prompt": + return rpImg2Img({ imageB64: imageBase64, prompt: instruction, ...params }); // SD 3.5 img2img (#10) + case "replace-sky": + // True sky replacement is masked inpaint (#9). Without a mask (no in-app sky + // segmentation yet) degrade to a low-strength img2img (#10) so the foreground + // is mostly preserved. + if (maskBase64) + return rpInpaint({ + imageB64: imageBase64, + maskB64: maskBase64, + prompt: instruction, + ...params, + }); + return rpImg2Img({ + imageB64: imageBase64, + prompt: instruction, + ...params, + strength: params.strength ?? 0.4, + }); + case "magic-eraser": + case "generative-fill": + // SD 3.5 masked inpaint (#9) — white in the mask = the region to regenerate. + if (!maskBase64) throw new AiError("This edit needs a mask/selection.", 400); + return rpInpaint({ + imageB64: imageBase64, + maskB64: maskBase64, + prompt: instruction, + ...params, + }); + default: + throw new AiError("Unsupported operation.", 400); + } +} + +class AiError extends Error { + status: number; + constructor(message: string, status = 502) { + super(message); + this.status = status; + } +} + +// --------------------------------------------------------------------------- +// Backend: local Stable Diffusion (Automatic1111 / Forge / SD.Next img2img API) +// --------------------------------------------------------------------------- +async function editLocal(instruction: string, imageBase64: string): Promise { + const base = process.env.LOCAL_SD_URL; + if (!base || !/^https?:\/\//i.test(base)) { + throw new AiError("LOCAL_SD_URL is not a valid http(s) URL.", 500); + } + const url = `${base.replace(/\/$/, "")}/sdapi/v1/img2img`; + let res: Response; + try { + res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + init_images: [imageBase64], + prompt: instruction, + denoising_strength: Number(process.env.LOCAL_SD_DENOISE ?? 0.55), + steps: Number(process.env.LOCAL_SD_STEPS ?? 25), + cfg_scale: 7, + sampler_name: process.env.LOCAL_SD_SAMPLER || "Euler a", + }), + cache: "no-store", + }); + } catch { + throw new AiError("Could not reach your local Stable Diffusion server (LOCAL_SD_URL).", 502); + } + if (!res.ok) { + throw new AiError(`Local SD server error (${res.status}).`, 502); + } + const data = (await res.json().catch(() => null)) as { images?: string[] } | null; + const out = data?.images?.[0]; + if (!out) throw new AiError("Local SD server did not return an image.", 502); + // A1111 returns raw base64 PNG (no data: prefix). + return { imageBase64: out.includes(",") ? out.split(",")[1]! : out, mimeType: "image/png" }; +} + +// --------------------------------------------------------------------------- +// Backend: Hugging Face Inference API — instruction image editing. +// --------------------------------------------------------------------------- +async function editHuggingFace(instruction: string, imageBase64: string): Promise { + const token = process.env.HF_API_TOKEN; + if (!token) throw new AiError("HF_API_TOKEN is not set.", 500); + const model = process.env.HF_IMAGE_MODEL || "timbrooks/instruct-pix2pix"; + let res: Response; + try { + res = await fetch(`https://api-inference.huggingface.co/models/${model}`, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + // Wait for the model to warm up instead of a fast 503. + "x-wait-for-model": "true", + }, + body: JSON.stringify({ + inputs: imageBase64, + parameters: { prompt: instruction, guidance_scale: 7, image_guidance_scale: 1.5 }, + }), + cache: "no-store", + }); + } catch { + throw new AiError("Could not reach the Hugging Face Inference API.", 502); + } + if (!res.ok) { + // Truncated on purpose — never surface a full upstream body. + const detail = (await res.text().catch(() => "")).slice(0, 160); + if (res.status === 503) throw new AiError("The model is loading — try again in ~20s.", 503); + throw new AiError(`Hugging Face error (${res.status}). ${detail}`, 502); + } + // Success returns raw image bytes. + const outMime = res.headers.get("content-type") || "image/png"; + if (outMime.startsWith("application/json")) { + const j = (await res.json().catch(() => null)) as { error?: string } | null; + throw new AiError( + j?.error ? `Hugging Face: ${j.error}` : "Hugging Face returned no image.", + 502, + ); + } + const buf = await res.arrayBuffer(); + return { imageBase64: Buffer.from(buf).toString("base64"), mimeType: outMime }; +} + +// --------------------------------------------------------------------------- +// Backend: Google Gemini image model (needs a billed key for image output). +// --------------------------------------------------------------------------- +async function editGemini( + instruction: string, + imageBase64: string, + safeMime: string, +): Promise { + const apiKey = process.env.GEMINI_API_KEY; + if (!apiKey) throw new AiError("GEMINI_API_KEY is not set.", 500); + const model = process.env.GEMINI_IMAGE_MODEL || "gemini-2.5-flash-image"; + const prompt = `Edit this image as follows: ${instruction}. Preserve realism unless explicitly asked otherwise.`; + let res: Response; + try { + res = await fetch( + `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, + { + method: "POST", + headers: { "content-type": "application/json", "x-goog-api-key": apiKey }, + body: JSON.stringify({ + contents: [ + { + role: "user", + parts: [ + { inlineData: { mimeType: safeMime, data: imageBase64 } }, + { text: prompt }, + ], + }, + ], + generationConfig: { responseModalities: ["IMAGE"] }, + }), + cache: "no-store", + }, + ); + } catch { + throw new AiError("Could not reach the AI service.", 502); + } + if (!res.ok) { + // Truncated on purpose — never surface a full upstream body. + const detail = (await res.text().catch(() => "")).slice(0, 160); + throw new AiError(`AI service error (${res.status}). ${detail}`, 502); + } + const data = (await res.json().catch(() => null)) as GeminiResponse | null; + const parts = data?.candidates?.[0]?.content?.parts ?? []; + const imgPart = parts.find((p) => p.inlineData?.data || p.inline_data?.data); + const out = imgPart?.inlineData?.data ?? imgPart?.inline_data?.data; + if (!out) + throw new AiError( + "The model did not return an image (the free Gemini tier has no image output — use LOCAL_SD_URL or HF_API_TOKEN instead).", + 502, + ); + const outMime = imgPart?.inlineData?.mimeType ?? imgPart?.inline_data?.mime_type ?? "image/png"; + return { imageBase64: out, mimeType: outMime }; +} + +interface GeminiPart { + text?: string; + inlineData?: { mimeType?: string; data?: string }; + inline_data?: { mime_type?: string; data?: string }; +} +interface GeminiResponse { + candidates?: Array<{ content?: { parts?: GeminiPart[] } }>; +} diff --git a/src/app/api/gallery/ai/tilt/route.ts b/src/app/api/gallery/ai/tilt/route.ts new file mode 100644 index 0000000..03cb300 --- /dev/null +++ b/src/app/api/gallery/ai/tilt/route.ts @@ -0,0 +1,48 @@ +import { type NextRequest, NextResponse } from "next/server"; + +import { RunpodError } from "@/lib/server/runpod/client"; +import { rpTilt } from "@/lib/server/runpod/endpoints"; + +import { guard } from "../_guard"; + +export const runtime = "nodejs"; +export const maxDuration = 60; // cold-start tilt worker can take a while +export const dynamic = "force-dynamic"; + +/** + * Camera-tilt estimation proxy. Accepts a base64 image and returns + * {rollDegrees, pitchDegrees, fovDegrees} from the RunPod tilt endpoint + * (RUNPOD_TILT_URL) so the editor can auto-straighten. + * + * POST { image } -> { rollDegrees, pitchDegrees, fovDegrees } + * Auth: session-gated. Rate limit: 30/min. + */ + +const MAX_BASE64 = 4_000_000; // ~3 MB decoded + +export async function POST(req: NextRequest) { + const gate = await guard(req, "tilt"); + if (!gate.ok) return gate.response; + + let body: { image?: unknown }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 }); + } + + const image = typeof body.image === "string" ? body.image : ""; + if (!image) return NextResponse.json({ error: "Missing image." }, { status: 400 }); + if (image.length > MAX_BASE64) { + return NextResponse.json({ error: "Image too large." }, { status: 413 }); + } + + try { + const tilt = await rpTilt(image); + return NextResponse.json(tilt); + } catch (e) { + const msg = + e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Tilt estimate failed."; + return NextResponse.json({ error: msg }, { status: 502 }); + } +} diff --git a/src/app/api/gallery/ai/transcribe/route.ts b/src/app/api/gallery/ai/transcribe/route.ts new file mode 100644 index 0000000..8428024 --- /dev/null +++ b/src/app/api/gallery/ai/transcribe/route.ts @@ -0,0 +1,49 @@ +import { type NextRequest, NextResponse } from "next/server"; + +import { RunpodError } from "@/lib/server/runpod/client"; +import { rpTranscribe } from "@/lib/server/runpod/endpoints"; + +import { guard } from "../_guard"; + +export const runtime = "nodejs"; +export const maxDuration = 60; // cold-start STT worker can take a while +export const dynamic = "force-dynamic"; + +/** + * Speech-to-text proxy for voice annotations. Accepts base64 WAV (16 kHz mono + * PCM16, produced in-browser) and returns the transcript. Calls the RunPod + * voice-to-text endpoint (RUNPOD_STT_URL) — the key stays server-side. + * + * POST { audio, language? } -> { transcript, segments? } + * Auth: session-gated. Rate limit: 20/min. + */ + +const MAX_BASE64 = 8_000_000; // ~6 MB decoded WAV — stays under serverless body limits + +export async function POST(req: NextRequest) { + const gate = await guard(req, "transcribe"); + if (!gate.ok) return gate.response; + + let body: { audio?: unknown; language?: unknown }; + try { + body = await req.json(); + } catch { + return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 }); + } + + const audio = typeof body.audio === "string" ? body.audio : ""; + const language = typeof body.language === "string" ? body.language : undefined; + if (!audio) return NextResponse.json({ error: "Missing audio." }, { status: 400 }); + if (audio.length > MAX_BASE64) { + return NextResponse.json({ error: "Audio too long — keep it under ~30s." }, { status: 413 }); + } + + try { + const { transcript, segments } = await rpTranscribe(audio, { language, punctuation: true }); + return NextResponse.json({ transcript, segments }); + } catch (e) { + const msg = + e instanceof RunpodError ? e.message : e instanceof Error ? e.message : "Transcription failed."; + return NextResponse.json({ error: msg }, { status: 502 }); + } +} diff --git a/src/app/dashboard/dashboard.css b/src/app/dashboard/dashboard.css index de8381a..ab10e56 100644 --- a/src/app/dashboard/dashboard.css +++ b/src/app/dashboard/dashboard.css @@ -1140,4 +1140,80 @@ .dash-root .ai-bubble { max-width: 86%; } .dash-root .ai-view { height: calc(100vh - 150px); } } - \ No newline at end of file + +/* ========================================================================= + Smart Gallery — the embedded @photo-gallery/sdk surface. + The SDK is themed entirely through the token map in lib/gallery-api.ts + (--apg-* -> this file's own vars), so the rules below only handle the + host chrome: sizing, the demo banner, and the load/error placeholders. + ========================================================================= */ + +/* The gallery is the one view that wants the whole viewport: it has its own sidebar, toolbar and + scrollers, so any height we leave on the table is wasted chrome. The old big PageHead cost ~90px; + a slim header (~40px) + compact banner + tight gaps hand almost all of that back to the shell. + 96px = the dashboard's top padding + the slim header row; `.gal-shell` (flex:1; min-height:0) + consumes whatever is left after the header and the optional demo banner. */ +.dash-root .gal { display: flex; flex-direction: column; gap: 10px; height: calc(100vh - 96px); min-height: 700px; } + +/* Slim inline header — replaces the tall PageHead. One row, ~40px, so the shell keeps the height. */ +.dash-root .gal-head { display: flex; align-items: center; gap: 10px; min-height: 36px; flex: 0 0 auto; } +.dash-root .gal-head-ic { width: 28px; height: 28px; border-radius: 9px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); flex: 0 0 auto; } +.dash-root .gal-head-title { font-size: 16px; font-weight: 700; line-height: 1.1; margin: 0; } +.dash-root .gal-head-sub { color: var(--muted); font-size: 12.5px; font-weight: 500; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; min-width: 0; } +@media (max-width: 720px) { .dash-root .gal-head-sub { display: none; } } + +/* `.view` sets `z-index: 1`, which makes it a stacking context and traps the SDK's + full-screen overlays (lightbox z1000, editors z1050, camera z1080, modals z1100) + underneath the topbar's `z-index: 20`. Opting this one view out of the stacking + context lets those overlays cover the whole dashboard, as they must. The view still + paints above the ambient `.dash-content::before` glow because it follows it in the DOM. */ +.dash-root .view.gal { z-index: auto; } + +/* Compact single-line demo banner (~34px). Truncates rather than wrapping so it never steals a + second row of height from the shell. */ +.dash-root .gal-banner { display: flex; align-items: center; gap: 8px; min-height: 34px; padding: 6px 12px; border-radius: 11px; border: 1px solid var(--border); background: color-mix(in srgb, var(--orange) 9%, var(--panel-2)); color: var(--text-2); font-size: 12px; font-weight: 500; flex: 0 0 auto; white-space: nowrap; overflow: hidden; } +.dash-root .gal-banner span { overflow: hidden; text-overflow: ellipsis; } +.dash-root .gal-banner svg { color: var(--orange); flex: 0 0 auto; } + +/* The gallery's own viewport. `overflow: hidden` keeps the SDK's internal scrollers + in charge; its full-screen overlays (lightbox/editor/camera) are position:fixed + and deliberately escape this box to cover the whole dashboard. */ +.dash-root .gal-shell { flex: 1; min-height: 0; position: relative; border-radius: 18px; border: 1px solid var(--border); background: var(--panel-2); overflow: hidden; box-shadow: var(--card-hi), 0 1px 2px rgba(0, 0, 0, 0.18); } +.dash-root[data-theme="dark"] .gal-shell { border: 0.5px solid #452b1a; border-radius: 20px; } + +/* The SDK's embedded root fills this box. (--apg-overlay-top is set from the + component's `style` prop — the SDK writes an inline default that a stylesheet + rule could not override.) */ +.dash-root .gal-shell .apg { height: 100%; } + +/* ---- Fullscreen (the SDK puts `.apg--fullscreen` on its root: position:fixed; inset:0) ---- + A position:fixed box is only clipped by an ancestor that is its CONTAINING BLOCK, which + `overflow`/`border-radius`/`box-shadow` alone never create — only transform / filter / + perspective / backdrop-filter / will-change / contain do. Nothing on the path + (.dash-content > .view.gal > .gal-shell) uses any of those: `.view`'s `ds-fade` animates + opacity only, and `.view.gal` already drops the `z-index: 1` stacking context. So the + fullscreen root does escape today — these rules make that survive an edit above. */ + +/* `.dash-root .gal-shell .apg` (0,3,0) would otherwise out-specify the SDK's own sizing; with + inset:0 driving the box, height must get out of the way. */ +.dash-root .gal-shell .apg.apg--fullscreen { + height: auto; + /* Above the topbar (z-index: 20) and the sidebar, below the SDK's own overlays (1000+). */ + z-index: 900; +} + +/* Belt and braces: if a future rule ever DOES make `.gal-shell` a containing block, an + `overflow: hidden` on it would crop the fullscreen root to the embedded box. Drop the clip + (and the rounded corner it exists to enforce) for exactly as long as fullscreen is on. */ +.dash-root .gal-shell:has(.apg--fullscreen) { overflow: visible; } + +.dash-root .gal-placeholder { height: 100%; min-height: 320px; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 12px; text-align: center; padding: 28px; } +.dash-root .gal-placeholder-ic { width: 66px; height: 66px; border-radius: 20px; display: grid; place-items: center; color: #fff; background: var(--grad-brand); box-shadow: var(--glow-orange); } +.dash-root .gal-placeholder p { color: var(--muted); font-size: 13px; max-width: 420px; } +.dash-root .gal-placeholder h3 { font-size: 16px; font-weight: 700; } +.dash-root .gal-placeholder-error .gal-placeholder-ic { background: color-mix(in srgb, var(--red) 88%, #000); box-shadow: 0 10px 28px -12px color-mix(in srgb, var(--red) 60%, transparent); } + +@media (max-width: 920px) { + /* Narrower chrome: a little less top offset, and a smaller floor so short viewports still work. */ + .dash-root .gal { height: calc(100vh - 84px); min-height: 560px; } +} diff --git a/src/components/dashboard/dashboard.tsx b/src/components/dashboard/dashboard.tsx index d052a4e..5c1cd24 100644 --- a/src/components/dashboard/dashboard.tsx +++ b/src/components/dashboard/dashboard.tsx @@ -17,6 +17,7 @@ import { AiAssistant } from "./ai-assistant"; import { TeamManagement } from "./team-management"; import { Messenger } from "./messenger"; import { Inbox } from "./inbox"; +import { SmartGallery } from "./smart-gallery"; import "../../app/dashboard/dashboard.css"; export function Dashboard() { @@ -49,6 +50,7 @@ export function Dashboard() { : active === "ai" ? : active === "messenger" ? : active === "inbox" ? + : active === "gallery" ? : active === "team" ? : } diff --git a/src/components/dashboard/sidebar.tsx b/src/components/dashboard/sidebar.tsx index e995cb0..527a000 100644 --- a/src/components/dashboard/sidebar.tsx +++ b/src/components/dashboard/sidebar.tsx @@ -34,6 +34,7 @@ export const NAV_GROUPS: NavGroup[] = [ items: [ { key: "messenger", label: "Messenger", icon: "send", subtitle: "Chat with your team and clients" }, { key: "inbox", label: "Inbox", icon: "bell", subtitle: "Mentions, messages, alerts and mail — all in one" }, + { key: "gallery", label: "Smart Gallery", icon: "gallery", subtitle: "Every photo and video for your jobs — searchable, editable and shareable" }, ], }, { @@ -79,7 +80,15 @@ export const NAV_ITEMS: NavItem[] = NAV_GROUPS.flatMap((g) => g.items); // brand-new user with no membership); every other item requires membership, and the // items mapped here additionally require the given permission. Unmapped items are // shown to any member. This is UX only — be-crm still enforces every action. -const ALWAYS_VISIBLE = new Set(["dashboard", "profile", "messenger", "inbox"]); +// +// `gallery` DECISION: it stays in ALWAYS_VISIBLE (nav row always shown to members) rather than +// being gated here by `media.view`. The real access gate lives INSIDE the view (SmartGallery reads +// `useGalleryFeatures().canView`), which uses the permissive "member with zero media.* perms → all +// enabled" fallback. Gating the nav row here would use the stricter sidebar rule (member without the +// perm → hidden) and so would hide the gallery from freshly-seeded members before an admin has +// configured any Media perms — regressing the demo and the common member. So: always-visible row, +// real gating in-view. (be-crm enforces data access regardless of what the nav shows.) +const ALWAYS_VISIBLE = new Set(["dashboard", "profile", "messenger", "inbox", "gallery"]); const NAV_PERMISSION: Record = { team: "team.manage", people: "team.manage", diff --git a/src/components/dashboard/smart-gallery-mount.tsx b/src/components/dashboard/smart-gallery-mount.tsx new file mode 100644 index 0000000..200a606 --- /dev/null +++ b/src/components/dashboard/smart-gallery-mount.tsx @@ -0,0 +1,69 @@ +"use client"; + +// ============================================================ +// The actual @photo-gallery/sdk mount. Split out of smart-gallery.tsx so it can be +// loaded with next/dynamic({ ssr: false }) — the SDK is browser-only (matchMedia, +// IndexedDB, Leaflet, canvas) and pulls in heavy optional ML models on demand, so it +// must stay out of the dashboard's initial bundle and out of the server render. +// ============================================================ + +import { useMemo } from "react"; +import { PhotoGallery, type ViewId } from "@photo-gallery/sdk"; +import { createCrmAIProvider } from "@/lib/gallery-ai"; +import { + GALLERY_THEME_TOKENS, + useGalleryFeatures, + useGalleryLockProvider, + useGalleryStorage, + useGalleryUser, +} from "@/lib/gallery-api"; + +import "@photo-gallery/sdk/styles.css"; +import "leaflet/dist/leaflet.css"; + +export interface SmartGalleryMountProps { + /** The dashboard's current appearance — the gallery must never diverge from the host. */ + theme: "dark" | "light"; +} + +export default function SmartGalleryMount({ theme }: SmartGalleryMountProps) { + const { adapter } = useGalleryStorage(); + const currentUser = useGalleryUser(); + // Server-backed Recently Deleted lock when the Shell is wired; `undefined` in demo mode, which + // leaves the SDK on its own device-local lock (see useGalleryLockProvider). + const lockProvider = useGalleryLockProvider(); + // Feature toggles resolved from the caller's CRM permissions (Media group). Superadmins/owners and + // the demo see everything; see useGalleryFeatures for the safe permissive fallback. + const { features } = useGalleryFeatures(); + + // Sidebar rows + Collections sections the CRM never wants to surface. The SDK hides both the row and + // the matching Collections section for each id. Screenshots + Documents aren't part of a roofing CRM. + const hiddenViews: ViewId[] = ["screenshots", "sys:documents"]; + + // One provider per mount. Every model is dynamically imported inside it, so constructing + // it is cheap; the weight only arrives when a photo is actually analyzed. + const ai = useMemo(() => createCrmAIProvider(), []); + + return ( + + ); +} diff --git a/src/components/dashboard/smart-gallery.tsx b/src/components/dashboard/smart-gallery.tsx new file mode 100644 index 0000000..c4e7f93 --- /dev/null +++ b/src/components/dashboard/smart-gallery.tsx @@ -0,0 +1,122 @@ +"use client"; + +// ============================================================ +// Smart Gallery — the tenant's media library, powered by @photo-gallery/sdk embedded +// inside the dashboard shell. The SDK owns the gallery experience (grid, lightbox, +// photo + video editors, map, people, versions, comments, AI); this file owns the +// LynkedUp chrome around it: page head, demo-mode banner, sizing, and failure +// containment so a gallery fault can never take the dashboard down. +// +// Storage + identity come from `@/lib/gallery-api` (be-crm data door when the Shell is +// configured, device-local otherwise). Theming comes from GALLERY_THEME_TOKENS, which +// maps the SDK's tokens onto this dashboard's own CSS variables. +// ============================================================ + +import { Component, type ErrorInfo, type ReactNode } from "react"; +import dynamic from "next/dynamic"; +import { Icon } from "./ui"; +import { useGalleryFeatures, useGalleryStorage } from "@/lib/gallery-api"; + +const SmartGalleryMount = dynamic(() => import("./smart-gallery-mount"), { + ssr: false, + loading: () => , +}); + +export function SmartGallery({ theme }: { theme: "dark" | "light" }) { + const { live } = useGalleryStorage(); + // Whole-view gate. `media.view` with the same permissive fallback the feature toggles use, so a + // superadmin/owner and the demo always pass, and a member is only blocked if they hold some Media + // perms but not `media.view`. The sidebar keeps the row visible (see sidebar.tsx §4) — the real + // gate is here. + const { canView } = useGalleryFeatures(); + + return ( +
+ {/* Slim header — the big PageHead ate vertical space the gallery needs. The CRM topbar already + shows the "Smart Gallery" title; this compact row (~40px) just adds context and the icon. */} +
+ + + +

Smart Gallery

+ Every photo and video for your jobs — searchable, editable and shareable. +
+ + {!live && ( +
+ + Demo mode — stored on this device only. It goes live once the Shell + be-crm are connected. +
+ )} + + {canView ? ( +
+ + + +
+ ) : ( +
+
+ + + +

You don't have access to the gallery

+

Ask a workspace admin to grant you the “View Smart Gallery” permission.

+
+
+ )} +
+ ); +} + +/* ------------------------------------------------------------------ */ + +function GalleryPlaceholder({ label }: { label: string }) { + return ( +
+ + + +

{label}

+
+ ); +} + +interface BoundaryState { + error: Error | null; +} + +/** + * The gallery is a large third-party surface with canvas, workers and optional ML models. A render + * fault inside it must degrade to a message rather than blanking the whole dashboard, so it gets its + * own error boundary. (Error boundaries still require a class component in React 19.) + */ +class GalleryBoundary extends Component<{ children: ReactNode }, BoundaryState> { + state: BoundaryState = { error: null }; + + static getDerivedStateFromError(error: Error): BoundaryState { + return { error }; + } + + componentDidCatch(error: Error, info: ErrorInfo): void { + console.error("[smart-gallery] render failed", error, info.componentStack); + } + + render(): ReactNode { + const { error } = this.state; + if (!error) return this.props.children; + return ( +
+ + + +

The gallery could not be displayed

+

{error.message || "An unexpected error occurred."}

+ +
+ ); + } +} diff --git a/src/components/dashboard/ui.tsx b/src/components/dashboard/ui.tsx index 5b57735..db72ce0 100644 --- a/src/components/dashboard/ui.tsx +++ b/src/components/dashboard/ui.tsx @@ -24,7 +24,8 @@ import { LayoutDashboard, Building2, FolderKanban, UserPlus, BadgeCheck, Filter, Truck, CloudLightning, Map as MapIcon, PenTool, Calculator, CalendarDays, Trophy, ListChecks, Users, Settings, Sparkles, MoreHorizontal, - UsersRound, type LucideIcon, + UsersRound, Image as ImageIcon, Images, File, FolderOpen, Download, + Video, Play, LayoutGrid, ZoomIn, type LucideIcon, } from "lucide-react"; /* ---------------------------------------------------------- */ @@ -50,6 +51,10 @@ const ICONS: Record = { estimates: Calculator, schedule: CalendarDays, leaderboard: Trophy, subtasks: ListChecks, people: Users, settings: Settings, ai: Sparkles, team: UsersRound, dots: MoreHorizontal, + // media / gallery (also used by messenger.tsx, which already asks for image/file) + image: ImageIcon, gallery: Images, file: File, folder: FolderOpen, + download: Download, video: Video, play: Play, grid: LayoutGrid, + filter: Filter, zoom: ZoomIn, sparkle: Sparkles, "map-pin": MapPin, }; export function Icon({ name, size = 18, className, strokeWidth = 2 }: { name: string; size?: number; className?: string; strokeWidth?: number }) { diff --git a/src/lib/gallery-ai.ts b/src/lib/gallery-ai.ts new file mode 100644 index 0000000..d0d7f68 --- /dev/null +++ b/src/lib/gallery-ai.ts @@ -0,0 +1,734 @@ +"use client"; + +/** + * The CRM's AIProvider for the Smart Gallery. + * + * PROVENANCE: a port of the SDK demo's `createDemoAIProvider()` plus its + * clip/face/ocr/tensorflow/runpod-yolo providers and `imageEncode.ts` helpers + * (advance-photo-gallery-web-sdk/apps/web/src/lib/ai/*), collapsed into one + * module and retargeted from `/api/ai/*` to `/api/gallery/ai/*`. + * + * Capability split: + * - object detection: TensorFlow.js COCO-SSD, fully in-browser (no key), or the + * RunPod YOLO classifier via /api/gallery/ai/classify when + * NEXT_PUBLIC_APG_RUNPOD_DETECT=true (COCO-SSD is the automatic fallback) + * - face detection + recognition: face-api.js in-browser → clustered into People + * - OCR: tesseract.js in-browser → searchable text + the Documents album + * - semantic search: CLIP via transformers.js in-browser + * - background removal: @imgly in-browser WASM, or the RunPod U²-Net endpoint + * when NEXT_PUBLIC_APG_RUNPOD_BG=true (in-browser is the fallback) + * - other generative edits / transcription / denoise / tilt: proxied through the + * server routes so the RunPod key never reaches the browser + * + * EVERY heavy model is behind `await import(...)` so none of it lands in the + * initial bundle, and every capability degrades to []/''/null with a + * console.warn rather than throwing — a failed model must never break the + * gallery UI. + * + * The in-browser models fetch weights from public CDNs (jsdelivr, huggingface, + * storage.googleapis.com, staticimgly.com). See docs/SMART_GALLERY.md for the + * list that would need CSP allow-listing. + */ + +import type { AIProvider, GenerativeEditOp, MediaItem } from "@photo-gallery/sdk"; + +// Derived from the provider interface so we import only the three public types. +type DetectedObject = Awaited>>[number]; +type DetectedFace = Awaited>>[number]; +type ImageSource = ImageBitmap | HTMLImageElement; + +const API = "/api/gallery/ai"; + +// --------------------------------------------------------------------------- +// imageEncode helpers (ported from apps/web/src/lib/ai/imageEncode.ts) +// --------------------------------------------------------------------------- + +export interface EncodedImage { + /** base64 JPEG (no data: prefix). */ + data: string; + mimeType: string; + /** Actual pixel dims of the encoded image (after downscale). */ + width: number; + height: number; +} + +/** Draw an image to a downscaled canvas and return base64 JPEG + its dims. */ +export function imageToBase64(image: ImageSource, maxDim: number): EncodedImage { + const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width; + const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height; + const scale = Math.min(1, maxDim / Math.max(w, h)); + const cw = Math.max(1, Math.round(w * scale)); + const ch = Math.max(1, Math.round(h * scale)); + const canvas = document.createElement("canvas"); + canvas.width = cw; + canvas.height = ch; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Canvas not supported."); + ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch); + const dataUrl = canvas.toDataURL("image/jpeg", 0.9); + return { data: dataUrl.split(",")[1] ?? "", mimeType: "image/jpeg", width: cw, height: ch }; +} + +/** + * Rasterize a mask (ImageData, white = region to regenerate) to a PNG base64 + * scaled to targetW×targetH so it matches the encoded image exactly — SD 3.5 + * requires image and mask to be identical pixel sizes. + */ +export function maskToBase64(mask: ImageData, targetW: number, targetH: number): string { + const tmp = document.createElement("canvas"); + tmp.width = mask.width; + tmp.height = mask.height; + const tctx = tmp.getContext("2d"); + if (!tctx) throw new Error("Canvas not supported."); + tctx.putImageData(mask, 0, 0); + + const out = document.createElement("canvas"); + out.width = targetW; + out.height = targetH; + const octx = out.getContext("2d"); + if (!octx) throw new Error("Canvas not supported."); + // Nearest-neighbour, not bilinear — keep the mask strictly binary so SD gets + // crisp white(regenerate)/black(keep) edges instead of an anti-aliased grey halo. + octx.imageSmoothingEnabled = false; + octx.drawImage(tmp, 0, 0, targetW, targetH); + return out.toDataURL("image/png").split(",")[1] ?? ""; +} + +export function base64ToBlob(base64: string, mime: string): Blob { + const bin = atob(base64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + return new Blob([bytes], { type: mime }); +} + +/** + * Pad an image with a neutral border for outpaint and return {imageBase64, maskBase64} + * as base64 PNG — the border is WHITE in the mask (regenerate), the original image + * area BLACK (keep). Capped at 1280px on the long side. + */ +export function padForOutpaint( + image: ImageSource, + factor: number, +): { imageBase64: string; maskBase64: string } { + const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width; + const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height; + const f = Math.max(1.1, Math.min(2, factor)); + const maxDim = 1280; + let pw = Math.round(w * f); + let ph = Math.round(h * f); + const scale = Math.min(1, maxDim / Math.max(pw, ph)); + pw = Math.max(16, Math.round(pw * scale)); + ph = Math.max(16, Math.round(ph * scale)); + const iw = Math.max(1, Math.round(w * scale)); + const ih = Math.max(1, Math.round(h * scale)); + const ox = Math.floor((pw - iw) / 2); + const oy = Math.floor((ph - ih) / 2); + + const imgCanvas = document.createElement("canvas"); + imgCanvas.width = pw; + imgCanvas.height = ph; + const ictx = imgCanvas.getContext("2d"); + if (!ictx) throw new Error("Canvas not supported."); + // Fill the new border with a blurred, stretched copy of the photo so the model + // has real color/context to continue from — flat gray gives it nothing. + ictx.filter = "blur(28px)"; + ictx.drawImage(image as CanvasImageSource, 0, 0, pw, ph); + ictx.filter = "none"; + ictx.drawImage(image as CanvasImageSource, ox, oy, iw, ih); + + const maskCanvas = document.createElement("canvas"); + maskCanvas.width = pw; + maskCanvas.height = ph; + const mctx = maskCanvas.getContext("2d"); + if (!mctx) throw new Error("Canvas not supported."); + mctx.fillStyle = "#ffffff"; + mctx.fillRect(0, 0, pw, ph); + mctx.fillStyle = "#000000"; + mctx.fillRect(ox, oy, iw, ih); + + return { + imageBase64: imgCanvas.toDataURL("image/png").split(",")[1] ?? "", + maskBase64: maskCanvas.toDataURL("image/png").split(",")[1] ?? "", + }; +} + +/** Draw an image to a canvas (downscaled) and return a JPEG Blob. */ +function canvasBlob(image: ImageSource, maxDim: number): Promise { + const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width; + const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height; + const scale = Math.min(1, maxDim / Math.max(w, h)); + const cw = Math.max(1, Math.round(w * scale)); + const ch = Math.max(1, Math.round(h * scale)); + const canvas = document.createElement("canvas"); + canvas.width = cw; + canvas.height = ch; + const ctx = canvas.getContext("2d"); + if (!ctx) return Promise.reject(new Error("Canvas not supported.")); + ctx.drawImage(image as CanvasImageSource, 0, 0, cw, ch); + return new Promise((resolve, reject) => + canvas.toBlob((b) => (b ? resolve(b) : reject(new Error("toBlob failed"))), "image/jpeg", 0.92), + ); +} + +function clamp01(n: number): number { + return n < 0 ? 0 : n > 1 ? 1 : n; +} + +// --------------------------------------------------------------------------- +// Object detection — TensorFlow.js COCO-SSD, in-browser +// --------------------------------------------------------------------------- + +interface CocoPrediction { + bbox: [number, number, number, number]; + class: string; + score: number; +} +interface CocoModel { + detect( + img: HTMLImageElement | HTMLCanvasElement | HTMLVideoElement, + maxNumBoxes?: number, + minScore?: number, + ): Promise; +} + +let cocoPromise: Promise | null = null; + +/** Load tfjs + COCO-SSD exactly once; resolves to null if anything fails. */ +function ensureCoco(): Promise { + cocoPromise ??= (async () => { + try { + const tf = await import("@tensorflow/tfjs"); + try { + await tf.setBackend("webgl"); + } catch { + // Fall back to the default backend if WebGL is unavailable. + } + await tf.ready(); + const cocoSsd = await import("@tensorflow-models/coco-ssd"); + return (await cocoSsd.load({ base: "lite_mobilenet_v2" })) as unknown as CocoModel; + } catch (err) { + console.warn("[gallery-ai] COCO-SSD load failed; object detection disabled.", err); + return null; + } + })(); + return cocoPromise; +} + +async function detectObjectsInBrowser( + item: MediaItem, + image: ImageSource, +): Promise { + const model = await ensureCoco(); + if (!model) return []; + try { + const el = image as HTMLImageElement; + const w = el.naturalWidth || el.width || item.width || 1; + const h = el.naturalHeight || el.height || item.height || 1; + const predictions = await model.detect(el, 20, 0.4); + return predictions.map((p) => ({ + label: p.class, + confidence: p.score, + box: { x: p.bbox[0] / w, y: p.bbox[1] / h, width: p.bbox[2] / w, height: p.bbox[3] / h }, + })) as DetectedObject[]; + } catch (err) { + console.warn("[gallery-ai] object detection failed.", err); + return []; + } +} + +/** + * Server-side YOLO detection via /api/gallery/ai/classify, with the in-browser + * COCO-SSD as the automatic fallback so detection never hard-fails. + */ +async function detectObjectsViaRunpod( + item: MediaItem, + image: ImageSource, +): Promise { + try { + const { data, mimeType, width, height } = imageToBase64(image, 1280); + const res = await fetch(`${API}/classify`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ imageBase64: data, mimeType, width, height }), + }); + if (!res.ok) throw new Error(`classify failed (${res.status})`); + const { objects } = (await res.json()) as { objects?: DetectedObject[] }; + if (Array.isArray(objects)) return objects; + throw new Error("classify returned no objects"); + } catch (err) { + console.warn("[gallery-ai] RunPod detection failed; falling back to COCO-SSD.", err); + return detectObjectsInBrowser(item, image); + } +} + +// --------------------------------------------------------------------------- +// Faces — @vladmandic/face-api, in-browser (128-D descriptors → People) +// --------------------------------------------------------------------------- + +const FACE_MODEL_URL = "https://cdn.jsdelivr.net/npm/@vladmandic/face-api@1.7.15/model"; + +type FaceApi = typeof import("@vladmandic/face-api"); + +let facePromise: Promise | null = null; + +function ensureFaceModels(): Promise { + facePromise ??= (async () => { + try { + const faceapi = await import("@vladmandic/face-api"); + // The bundled tf re-export is typed narrowly; backend control lives on the + // runtime object. Prefer WebGL (no eval; CSP-friendly), fall back gracefully. + const tf = faceapi.tf as unknown as { + setBackend: (b: string) => Promise; + ready: () => Promise; + }; + try { + await tf.setBackend("webgl"); + } catch { + /* keep default backend */ + } + await tf.ready(); + await Promise.all([ + faceapi.nets.tinyFaceDetector.loadFromUri(FACE_MODEL_URL), + faceapi.nets.faceLandmark68Net.loadFromUri(FACE_MODEL_URL), + faceapi.nets.faceRecognitionNet.loadFromUri(FACE_MODEL_URL), + ]); + return faceapi; + } catch (err) { + // Degrade gracefully — People simply stays empty if models can't load. + console.warn("[gallery-ai] face model load failed; face clustering disabled.", err); + return null; + } + })(); + return facePromise; +} + +let faceWarned = false; + +async function detectFaces(item: MediaItem, image: ImageSource): Promise { + const faceapi = await ensureFaceModels(); + if (!faceapi) return []; + + const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width || 1; + const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height || 1; + + type TNetInput = Parameters[0]; + + let results; + try { + results = await faceapi + .detectAllFaces( + image as unknown as TNetInput, + new faceapi.TinyFaceDetectorOptions({ inputSize: 416, scoreThreshold: 0.5 }), + ) + .withFaceLandmarks() + .withFaceDescriptors(); + } catch (err) { + if (!faceWarned) { + faceWarned = true; + console.warn("[gallery-ai] face detection failed on", item.name, err); + } + return []; + } + + return results.map((r) => { + const b = r.detection.box; + return { + confidence: r.detection.score, + box: { + x: clamp01(b.x / w), + y: clamp01(b.y / h), + width: clamp01(b.width / w), + height: clamp01(b.height / h), + }, + embedding: Array.from(r.descriptor), + }; + }) as DetectedFace[]; +} + +// --------------------------------------------------------------------------- +// OCR — tesseract.js, in-browser +// --------------------------------------------------------------------------- + +// Must equal the EXACT tesseract.js version in package.json (pinned, no caret) +// so the worker CDN URL can never drift from the installed main-thread code. +const TESSERACT_VERSION = "5.1.1"; +const WORKER_PATH = `https://cdn.jsdelivr.net/npm/tesseract.js@${TESSERACT_VERSION}/dist/worker.min.js`; +const CORE_PATH = "https://cdn.jsdelivr.net/npm/tesseract.js-core@5"; +// jsDelivr's GitHub mirror of naptha/tessdata (same files as projectnaptha.com), +// so every asset comes from ONE host that a CSP can allow-list. +const LANG_PATH = "https://cdn.jsdelivr.net/gh/naptha/tessdata@gh-pages/4.0.0"; + +interface OcrWord { + text?: string; + confidence?: number; +} +interface OcrData { + text?: string; + confidence?: number; + words?: OcrWord[]; + blocks?: Array<{ paragraphs?: Array<{ lines?: Array<{ words?: OcrWord[] }> }> }> | null; +} + +type TesseractWorker = import("tesseract.js").Worker; + +let ocrWorkerPromise: Promise | null = null; + +function ensureOcrWorker(): Promise { + ocrWorkerPromise ??= (async () => { + try { + const { createWorker } = await import("tesseract.js"); + // v5: createWorker(langs, oem, options) already loads + initializes the + // language internally — do NOT call the removed v4 worker.load(). + return await createWorker("eng", 1, { + workerPath: WORKER_PATH, + corePath: CORE_PATH, + langPath: LANG_PATH, + }); + } catch (err) { + console.warn("[gallery-ai] tesseract worker init failed; OCR disabled.", err); + return null; + } + })(); + return ocrWorkerPromise; +} + +const WORD_CONFIDENCE = 70; // a word tesseract is actually sure about +const MIN_WORDS = 4; // need several confident words to call it a document +const MIN_CHARS = 10; + +function collectWords(data: OcrData): OcrWord[] { + if (Array.isArray(data.words) && data.words.length) return data.words; + const out: OcrWord[] = []; + for (const b of data.blocks ?? []) + for (const p of b.paragraphs ?? []) + for (const l of p.lines ?? []) for (const w of l.words ?? []) out.push(w); + return out; +} + +/** + * Return real text or '' (not a document). tesseract hallucinates low-confidence + * gibberish for photos with no text, so we keep only high-confidence, word-shaped + * tokens and require several of them. + */ +function meaningfulText(data: OcrData): string { + const words = collectWords(data); + if (words.length > 0) { + const good = words.filter( + (w) => (w.confidence ?? 0) >= WORD_CONFIDENCE && /[A-Za-z0-9]{2,}/.test(w.text ?? ""), + ); + const text = good + .map((w) => (w.text ?? "").trim()) + .filter(Boolean) + .join(" ") + .trim(); + return good.length >= MIN_WORDS && text.length >= MIN_CHARS ? text : ""; + } + // Fallback: overall confidence + count of word-shaped tokens. + const raw = (data.text ?? "").trim(); + const conf = typeof data.confidence === "number" ? data.confidence : 0; + const realWords = raw.match(/[A-Za-z]{3,}/g) ?? []; + return conf >= 72 && realWords.length >= 6 ? raw : ""; +} + +async function ocr(_item: MediaItem, image: ImageSource): Promise { + const worker = await ensureOcrWorker(); + if (!worker) return ""; + try { + // Request the block hierarchy so per-word confidence is available. + const { data } = (await worker.recognize( + image as unknown as HTMLImageElement, + {}, + { text: true, blocks: true }, + )) as { data: OcrData }; + return meaningfulText(data); + } catch (err) { + console.warn("[gallery-ai] OCR failed.", err); + return ""; + } +} + +// --------------------------------------------------------------------------- +// Semantic search — CLIP via transformers.js (ONNX-WASM), in-browser +// --------------------------------------------------------------------------- + +const CLIP_MODEL_ID = "Xenova/clip-vit-base-patch16"; + +type Transformers = typeof import("@huggingface/transformers"); + +let transformersMod: Transformers | null = null; +/* eslint-disable @typescript-eslint/no-explicit-any -- transformers.js pipelines are untyped */ +let clipVisionPromise: Promise<{ processor: any; model: any } | null> | null = null; +let clipTextPromise: Promise<{ tokenizer: any; model: any } | null> | null = null; + +async function loadTransformers(): Promise { + if (!transformersMod) { + transformersMod = await import("@huggingface/transformers"); + // Remote-only (models from the HF CDN); rely on browser cache between sessions. + transformersMod.env.allowLocalModels = false; + } + return transformersMod; +} + +function ensureClipVision() { + clipVisionPromise ??= (async () => { + try { + const tf = await loadTransformers(); + const [processor, model] = await Promise.all([ + tf.AutoProcessor.from_pretrained(CLIP_MODEL_ID), + tf.CLIPVisionModelWithProjection.from_pretrained(CLIP_MODEL_ID), + ]); + return { processor, model }; + } catch (err) { + console.warn("[gallery-ai] CLIP vision load failed; semantic search disabled.", err); + return null; + } + })(); + return clipVisionPromise; +} + +function ensureClipText() { + clipTextPromise ??= (async () => { + try { + const tf = await loadTransformers(); + const [tokenizer, model] = await Promise.all([ + tf.AutoTokenizer.from_pretrained(CLIP_MODEL_ID), + tf.CLIPTextModelWithProjection.from_pretrained(CLIP_MODEL_ID), + ]); + return { tokenizer, model }; + } catch (err) { + console.warn("[gallery-ai] CLIP text load failed; semantic search disabled.", err); + return null; + } + })(); + return clipTextPromise; +} + +/** Draw an image onto a canvas (downscaled) for the CLIP image processor. */ +function toCanvas(image: ImageSource, maxDim = 384): HTMLCanvasElement { + const w = (image as HTMLImageElement).naturalWidth || (image as ImageBitmap).width; + const h = (image as HTMLImageElement).naturalHeight || (image as ImageBitmap).height; + const scale = Math.min(1, maxDim / Math.max(w, h)); + const canvas = document.createElement("canvas"); + canvas.width = Math.max(1, Math.round(w * scale)); + canvas.height = Math.max(1, Math.round(h * scale)); + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Canvas not supported."); + ctx.drawImage(image as CanvasImageSource, 0, 0, canvas.width, canvas.height); + return canvas; +} + +function tensorToArray(t: any): number[] { + const data: Float32Array = t?.data ?? t; + return Array.from(data as ArrayLike); +} + +async function embedImage(_item: MediaItem, image: ImageSource): Promise { + const v = await ensureClipVision(); + if (!v) return []; + try { + const tf = await loadTransformers(); + const canvas = toCanvas(image); + const ctx = canvas.getContext("2d"); + if (!ctx) return []; + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const raw = new tf.RawImage(imageData.data, canvas.width, canvas.height, 4).rgb(); + const inputs = await v.processor(raw); + const out = await v.model(inputs); + return tensorToArray(out.image_embeds); + } catch (err) { + console.warn("[gallery-ai] embedImage failed.", err); + return []; + } +} + +async function embedText(query: string): Promise { + const t = await ensureClipText(); + if (!t) return []; + try { + const inputs = t.tokenizer([query], { padding: true, truncation: true }); + const out = await t.model(inputs); + return tensorToArray(out.text_embeds); + } catch (err) { + console.warn("[gallery-ai] embedText failed.", err); + return []; + } +} +/* eslint-enable @typescript-eslint/no-explicit-any */ + +// --------------------------------------------------------------------------- +// The provider +// --------------------------------------------------------------------------- + +/** `true` only for the literal string "true", matching the SDK demo's semantics. */ +function flag(v: string | undefined): boolean { + return v === "true"; +} + +export function createCrmAIProvider(): AIProvider { + // NEXT_PUBLIC_* are inlined at build time, so these must be read as full + // static member expressions — do NOT refactor to dynamic indexing. + const useRunpodDetect = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_DETECT); + const useRunpodBg = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_BG); + const useRunpodTilt = flag(process.env.NEXT_PUBLIC_APG_RUNPOD_TILT); + + return { + name: "crm-ai (coco-ssd/yolo + face-api + tesseract + clip + runpod-edit)", + + detectObjects: useRunpodDetect ? detectObjectsViaRunpod : detectObjectsInBrowser, + detectFaces, + ocr, + embedImage, + embedText, + + async generativeEdit(item: MediaItem, image: ImageSource, op: GenerativeEditOp) { + // Remove Background runs fully in-browser (no key) via @imgly — works even + // with no backend. Other ops go through the /api/gallery/ai/edit route. + if (op.type === "remove-background") { + // Prefer the RunPod U²-Net endpoint when enabled; fall back to in-browser + // @imgly if it's off or the request fails, so this always produces a result. + if (useRunpodBg) { + try { + const { data, mimeType } = imageToBase64(image, 1600); + const res = await fetch(`${API}/edit`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + imageBase64: data, + mimeType, + op: { type: "remove-background" }, + params: {}, + }), + }); + if (res.ok) { + const { imageBase64: out, mimeType: outMime } = (await res.json()) as { + imageBase64: string; + mimeType?: string; + }; + return base64ToBlob(out, outMime || "image/png"); + } + } catch { + /* fall through to the in-browser remover */ + } + } + const inputBlob = await canvasBlob(image, 1600); + const { removeBackground } = await import("@imgly/background-removal"); + return removeBackground(inputBlob, { output: { format: "image/png" } }); + } + + // Outpaint / expand-canvas: pad the image with a neutral border, mark that + // border WHITE in the mask, and run it through the same inpaint path as + // generative-fill — no extra backend route needed. + if (op.type === "outpaint") { + const { imageBase64: padded, maskBase64: border } = padForOutpaint( + image, + typeof op.factor === "number" ? op.factor : 1.5, + ); + const outParams: Record = { + strength: typeof op.strength === "number" ? op.strength : 0.85, + }; + const outRes = await fetch(`${API}/edit`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + imageBase64: padded, + mimeType: "image/png", + op: { + type: "generative-fill", + prompt: + op.prompt || + "Extend and continue the scene naturally, matching lighting, colors and perspective.", + }, + maskBase64: border, + params: outParams, + }), + }); + if (!outRes.ok) { + const err = (await outRes.json().catch(() => ({}))) as { error?: string }; + throw new Error(err.error || `AI request failed (${outRes.status}).`); + } + const outJson = (await outRes.json()) as { imageBase64: string; mimeType?: string }; + return base64ToBlob(outJson.imageBase64, outJson.mimeType || "image/png"); + } + + const { data, mimeType, width, height } = imageToBase64(image, 1280); + // Masked ops carry an ImageData mask — rasterize it to a PNG matched to the + // (downscaled) image dims, and strip it from the op since ImageData is not + // JSON-serializable. + const maskBase64 = "mask" in op ? maskToBase64(op.mask, width, height) : undefined; + const wireOp: Record = { type: op.type }; + if ("prompt" in op && typeof op.prompt === "string") wireOp.prompt = op.prompt; + if ("factor" in op && typeof op.factor === "number") wireOp.factor = op.factor; + + // Forward the "edit strength" slider (0..1) so the backend can scale the edit. + const params: Record = {}; + if ("strength" in op && typeof op.strength === "number") params.strength = op.strength; + + const res = await fetch(`${API}/edit`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ imageBase64: data, mimeType, op: wireOp, maskBase64, params }), + }); + if (!res.ok) { + const err = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(err.error || `AI request failed (${res.status}).`); + } + const { imageBase64, mimeType: outMime } = (await res.json()) as { + imageBase64: string; + mimeType?: string; + }; + return base64ToBlob(imageBase64, outMime || "image/png"); + }, + + // Voice annotation: record → (optional denoise) → transcribe. Both proxy + // through server routes so the RunPod key stays server-side. + async transcribeAudio(audioBase64: string) { + const res = await fetch(`${API}/transcribe`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ audio: audioBase64 }), + }); + if (!res.ok) { + const err = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(err.error || `Transcription failed (${res.status}).`); + } + const { transcript } = (await res.json()) as { transcript?: string }; + return (transcript ?? "").trim(); + }, + + async denoiseAudio(audioBase64: string) { + const res = await fetch(`${API}/denoise`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ audio: audioBase64 }), + }); + if (!res.ok) { + const err = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(err.error || `Denoise failed (${res.status}).`); + } + const { audio } = (await res.json()) as { audio?: string }; + return audio ?? audioBase64; + }, + + // Camera-tilt estimation is opt-in (needs the RunPod tilt endpoint deployed); + // gate it so the editor's Auto-straighten button only appears when configured. + estimateTilt: useRunpodTilt + ? async (_item: MediaItem, image: ImageSource) => { + const { data } = imageToBase64(image, 1024); + const res = await fetch(`${API}/tilt`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ image: data }), + }); + if (!res.ok) { + const err = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(err.error || `Tilt estimate failed (${res.status}).`); + } + return (await res.json()) as { + rollDegrees: number; + pitchDegrees: number; + fovDegrees: number; + }; + } + : undefined, + }; +} diff --git a/src/lib/gallery-api.ts b/src/lib/gallery-api.ts new file mode 100644 index 0000000..de33e85 --- /dev/null +++ b/src/lib/gallery-api.ts @@ -0,0 +1,418 @@ +"use client"; + +// Smart Gallery data layer. Serves EITHER a device-local mock (when the Shell isn't configured — the +// demo keeps working offline) OR the live be-crm data door (crm.gallery.*), behind one StorageAdapter +// so the embedded @photo-gallery/sdk is mode-agnostic. Tenant scoping, comment authorship and byte +// authorization are all enforced server-side; this is just glue. +// +// Live contract (be-crm): +// query crm.gallery.state.load {} -> PersistedState +// cmd crm.gallery.state.apply StateChanges -> { upserted, removed } +// cmd crm.gallery.media.presignUpload { mediaId, mime, sizeBytes, filename? } -> { ref, uploadUrl, method, headers? } +// cmd crm.gallery.media.presignDownload { refs: string[] } -> { urls, expiresInSeconds } +// query crm.gallery.stats {} -> { items, albums, people, bytes } +// query crm.gallery.lock.status {} -> { hasPassword } +// cmd crm.gallery.lock.set { password: string | null } -> { hasPassword } +// cmd crm.gallery.lock.verify { password: string } -> { ok } +// +// BYTES NEVER PASS THROUGH be-crm OR THE BFF. `putMedia` mints a short-lived signed PUT URL and the +// browser transfers straight to object storage — the same rule media-api.ts follows for attachments. + +import { useMemo } from "react"; +import { useAppShell, useAuth } from "@abe-kap/appshell-sdk/react"; +import { + createLocalStorageAdapter, + type GalleryFeatures, + type GalleryUser, + type MediaItem, + type PersistedState, + type StateChanges, + type StorageAdapter, + type StoredBlob, + type ThemeTokens, +} from "@photo-gallery/sdk"; +import { user as demoUser } from "@/components/dashboard/account-data"; +import { useMyAccess } from "@/lib/access"; +import { isShellConfigured } from "./appshell"; + +const SHELL = isShellConfigured(); + +/** Device-local store key for demo mode. Namespaced so it can't collide with the SDK's own demo. */ +const LOCAL_STORE_KEY = "lup:smart-gallery:v1"; + +/** be-crm caps a single presignDownload at 500 refs — chunk anything larger. */ +const PRESIGN_CHUNK = 500; + +/** Largest single upload the gallery will attempt (matches be-crm's gallery cap). */ +export const MAX_GALLERY_BYTES = 200 * 1024 * 1024; + +/* ======================================================================== */ +/* Wire types (be-crm shapes) */ +/* ======================================================================== */ + +interface StateLoadDTO { + media: MediaItem[]; + albums: PersistedState["albums"]; + people: PersistedState["people"]; + labelAliases?: Record; + deletedLabels?: string[]; +} + +interface PresignUploadDTO { + ref: string; + uploadUrl: string; + method: "PUT"; + headers?: Record; +} + +interface PresignDownloadDTO { + urls: Record; + expiresInSeconds: number; +} + +/** The subset of the AppShell SDK this module needs — keeps the adapter unit-testable. */ +interface DataDoor { + query(action: string, variables?: Record): Promise; + command(action: string, variables?: Record): Promise; +} + +/* ======================================================================== */ +/* The live adapter — be-crm data door */ +/* ======================================================================== */ + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + +/** + * A StorageAdapter backed by the be-crm data door. + * + * Metadata (media/albums/people) rides the door as JSON; bytes go direct to object storage via + * short-lived signed URLs. `storageRef` is the durable handle we persist — `src` is only ever a + * signed URL with a TTL, so it is re-resolved from the refs on every `load()`. + */ +export function createDataDoorAdapter(sdk: DataDoor): StorageAdapter { + /** Resolve durable refs → fresh signed GET URLs, in chunks the door will accept. */ + async function resolveRefs(refs: string[]): Promise> { + const unique = [...new Set(refs.filter(Boolean))]; + if (!unique.length) return {}; + const urls: Record = {}; + for (const group of chunk(unique, PRESIGN_CHUNK)) { + const res = await sdk.command("crm.gallery.media.presignDownload", { refs: group }); + Object.assign(urls, res.urls ?? {}); + } + return urls; + } + + return { + name: "crm-data-door", + + async load(): Promise { + const state = await sdk.query("crm.gallery.state.load", {}); + const media = state.media ?? []; + // Signed URLs expire, so `src` is always rebuilt from `storageRef` at load time. Items with no + // ref (e.g. a seeded remote URL) keep whatever `src` they were stored with. + const urls = await resolveRefs(media.map((m) => m.storageRef ?? "").filter(Boolean)); + return { + media: media.map((m) => (m.storageRef && urls[m.storageRef] ? { ...m, src: urls[m.storageRef]! } : m)), + albums: state.albums ?? [], + people: state.people ?? [], + labelAliases: state.labelAliases ?? {}, + deletedLabels: state.deletedLabels ?? [], + version: 1, + }; + }, + + // Incremental sync is the real path (see applyChanges). `save` only runs if the store ever falls + // back to whole-state persistence; express it as one big change set so behaviour is identical. + async save(state: PersistedState): Promise { + await sdk.command("crm.gallery.state.apply", { + upsertMedia: state.media, + upsertAlbums: state.albums, + upsertPeople: state.people, + labelAliases: state.labelAliases ?? {}, + deletedLabels: state.deletedLabels ?? [], + }); + }, + + /** + * Incremental persistence — only the entities that actually changed. This is what makes a shared + * tenant library safe for concurrent editors: two people touching different photos never clobber + * each other, because neither sends the other's rows. + */ + async applyChanges(changes: StateChanges): Promise { + // StateChanges is a closed interface; the door takes an open variables bag. + await sdk.command("crm.gallery.state.apply", { ...changes }); + }, + + /** Presign → direct PUT → resolve a display URL. Bytes never touch be-crm or the BFF. */ + async putMedia(id: string, blob: Blob, meta: { name: string; mime: string }): Promise { + if (blob.size > MAX_GALLERY_BYTES) { + throw new Error(`File is too large (max ${Math.floor(MAX_GALLERY_BYTES / (1024 * 1024))} MB).`); + } + const mime = meta.mime || blob.type || "application/octet-stream"; + const presigned = await sdk.command("crm.gallery.media.presignUpload", { + mediaId: id, + mime, + sizeBytes: blob.size, + filename: meta.name, + }); + const res = await fetch(presigned.uploadUrl, { + method: presigned.method ?? "PUT", + headers: { "content-type": mime, ...(presigned.headers ?? {}) }, + body: blob, + }); + if (!res.ok) throw new Error(`Upload failed (${res.status}).`); + const urls = await resolveRefs([presigned.ref]); + return { ref: presigned.ref, url: urls[presigned.ref] ?? presigned.ref }; + }, + }; +} + +/* ======================================================================== */ +/* Public hooks */ +/* ======================================================================== */ + +export interface GalleryStorage { + /** True when persisting to be-crm; false when running on the device-local demo store. */ + live: boolean; + adapter: StorageAdapter; +} + +/** + * The storage backend for the embedded gallery. Live against the be-crm data door once the Shell is + * configured, otherwise a device-local store so the demo works with no backend at all. + * + * SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe). + */ +export function useGalleryStorage(): GalleryStorage { + const { sdk } = useAppShell(); + // The adapter identity must be stable — PhotoGallery captures it in a ref on first render. + return useMemo( + () => + SHELL + ? { live: true, adapter: createDataDoorAdapter(sdk as unknown as DataDoor) } + : { live: false, adapter: createLocalStorageAdapter(LOCAL_STORE_KEY) }, + [sdk], + ); +} + +/** + * The signed-in identity handed to the gallery so comments are attributed to a real person rather + * than a free-text name the author can type themselves. + * + * Falls back to the same static demo user the sidebar and topbar use when the Shell isn't wired, so + * the comment module behaves identically in the demo — the CRM never shows an anonymous author. + * be-crm re-stamps `authorId` from the PAT on every write regardless, so this value is a display + * convenience, never the source of authority. + */ +export function useGalleryUser(): GalleryUser { + const { user, context } = useAuth(); + return useMemo(() => { + if (!user) return { id: demoUser.id, name: demoUser.name }; + const avatarUrl = user.avatarUrl ?? context?.principal?.avatarUrl; + return { + id: user.id, + name: user.displayName || user.email || demoUser.name, + ...(user.email ? { email: user.email } : {}), + ...(avatarUrl ? { avatarUrl } : {}), + }; + }, [user, context]); +} + +/* ======================================================================== */ +/* Recently Deleted lock */ +/* ======================================================================== */ + +/** + * The SDK's `lockProvider` contract, declared here rather than imported so this module keeps + * compiling against an SDK build that predates the prop. It is structurally identical to + * `PhotoGalleryProps['lockProvider']`. + */ +export interface GalleryLockProvider { + status(): Promise<{ hasPassword: boolean }>; + /** `null` clears the password. */ + set(password: string | null): Promise; + verify(password: string): Promise; +} + +/** + * A server-backed, per-user lock for the Recently Deleted view. + * + * Without this the SDK falls back to a device-local `localStorage` hash: the lock exists only on + * the browser that set it, so the same account on a second device sees an unlocked trash. Backed + * by `crm.gallery.lock.*`, the password becomes an account property — be-crm stores only a + * scrypt hash with a per-record random salt, keyed by (tenant, principal), and rate-limits verify. + * + * Returns `undefined` in demo mode ON PURPOSE: with no backend there is nowhere to put the hash, + * and the SDK's own device-local behaviour is the right fallback for a demo. + * + * SHELL is a build-time constant, so this branch is stable across renders (Rules-of-Hooks safe). + */ +export function useGalleryLockProvider(): GalleryLockProvider | undefined { + const { sdk } = useAppShell(); + return useMemo(() => { + if (!SHELL) return undefined; + const door = sdk as unknown as DataDoor; + return { + status: () => door.query<{ hasPassword: boolean }>("crm.gallery.lock.status", {}), + // The SDK's contract returns void; the door's `{ hasPassword }` is redundant after a set. + set: async (password) => { + await door.command("crm.gallery.lock.set", { password }); + }, + // A wrong password is a normal `{ ok: false }`, not an error. A 403 (the verify lockout) + // still throws, which is what the SDK's prompt should surface. + verify: async (password) => (await door.command<{ ok: boolean }>("crm.gallery.lock.verify", { password })).ok, + }; + }, [sdk]); +} + +/* ======================================================================== */ +/* Permission-gated features */ +/* ======================================================================== */ + +/** + * The gallery capabilities the CRM gates behind team permissions. Each SDK feature toggle + * (and the whole-view `media.view` gate) maps to one permission id from be-crm's Media group. + * The SDK still enforces nothing here — hiding a control is UX; be-crm enforces every write via + * GALLERY_VISIBILITY + tenant scoping regardless of what the UI shows. + */ +const MEDIA_PERMISSIONS = [ + "media.view", "media.upload", "media.capture", "media.edit", + "media.delete", "media.export", "media.share", "media.map", "media.ai", +] as const; + +/** SDK feature toggle → the permission id that unlocks it. */ +const FEATURE_PERMISSION: Record = { + editor: "media.edit", + camera: "media.capture", + import: "media.upload", + export: "media.export", + sharing: "media.share", + map: "media.map", + ai: "media.ai", +}; + +export interface ResolvedGalleryFeatures { + /** Feature toggles to hand the SDK's `features` prop, resolved from the caller's permissions. */ + features: GalleryFeatures; + /** Whether the whole Smart Gallery view should render at all (`media.view`). */ + canView: boolean; +} + +/** + * Resolve the gallery's feature flags + view access from the signed-in user's CRM permissions. + * + * Fallback rule (SAFE default — a control is enabled unless the user is positively known to lack it): + * 1. While access is still loading, stay permissive so features don't flash on then vanish. + * 2. Superadmins and owners get everything (they hold every permission anyway; this is explicit). + * 3. A member who has been assigned AT LEAST ONE `media.*` permission is gated precisely: each + * feature is enabled only if its mapped permission is in their list. + * 4. A member with ZERO `media.*` permissions assigned is treated as fully enabled. Roles are not + * configured with Media perms until an admin opts in (§5 just made them assignable), so gating a + * freshly-seeded member down to nothing would cripple the gallery before anyone could grant them + * anything. The "has at least one media.* perm" signal is what flips a role from this permissive + * default into precise per-feature gating. + * + * Because be-crm's mock access (demo, no Shell) returns superadmin + all perms, the demo shows + * everything via rule 2. + */ +export function useGalleryFeatures(): ResolvedGalleryFeatures { + const access = useMyAccess(); + return useMemo(() => { + const has = (p: string) => access.permissions.includes(p); + const privileged = access.isSuperadmin || access.roleSlugs.includes("owner"); + const hasAnyMedia = MEDIA_PERMISSIONS.some(has); + // Permissive whenever we can't (yet) prove the user lacks a permission: loading, privileged, or a + // member who has no Media perms assigned at all. Otherwise gate precisely on the mapped id. + const allow = (perm: string) => access.loading || privileged || !hasAnyMedia || has(perm); + + const features: GalleryFeatures = { + editor: allow(FEATURE_PERMISSION.editor), + camera: allow(FEATURE_PERMISSION.camera), + ai: allow(FEATURE_PERMISSION.ai), + map: allow(FEATURE_PERMISSION.map), + import: allow(FEATURE_PERMISSION.import), + export: allow(FEATURE_PERMISSION.export), + sharing: allow(FEATURE_PERMISSION.sharing), + }; + + return { features, canView: allow("media.view") }; + }, [access]); +} + +/* ======================================================================== */ +/* Theme bridge */ +/* ======================================================================== */ + +/** + * Maps the gallery's design tokens onto the CRM's own CSS variables. + * + * Every value is a `var(--crm-token)` reference rather than a literal hex, so the gallery inherits the + * dashboard's palette through the SAME `[data-theme]` cascade the rest of the app uses — light/dark + * switch together, and a future palette change reaches the gallery with no code edit here. + */ +export const GALLERY_THEME_TOKENS: ThemeTokens = { + // Surfaces + bgLight: "var(--bg)", + bgDark: "var(--bg)", + elevatedLight: "var(--panel-3)", + elevatedDark: "var(--panel-3)", + sidebarBgLight: "var(--sidebar)", + sidebarBgDark: "var(--sidebar)", + toolbarBgLight: "var(--panel)", + toolbarBgDark: "var(--panel)", + cardLight: "var(--panel)", + cardDark: "var(--panel)", + cardHoverLight: "var(--panel-3)", + cardHoverDark: "var(--panel-3)", + menuBgLight: "var(--panel-3)", + menuBgDark: "var(--panel-3)", + + // Text + textLight: "var(--text)", + textDark: "var(--text)", + textSecondaryLight: "var(--muted)", + textSecondaryDark: "var(--muted)", + textTertiaryLight: "var(--faint)", + textTertiaryDark: "var(--faint)", + + // Lines + washes + separatorLight: "var(--border)", + separatorDark: "var(--border)", + separatorStrongLight: "var(--border-2)", + separatorStrongDark: "var(--border-2)", + hoverLight: "var(--track)", + hoverDark: "var(--track)", + activeLight: "var(--border-2)", + activeDark: "var(--border-2)", + sidebarSelectedLight: "var(--track)", + sidebarSelectedDark: "var(--track)", + glassBorderLight: "var(--border-2)", + glassBorderDark: "var(--border-2)", + + // Brand + accent: "var(--orange)", + accentStrongLight: "var(--orange-2)", + accentStrongDark: "var(--orange-2)", + accentContrast: "#ffffff", + dangerLight: "var(--red)", + dangerDark: "var(--red)", + tileFav: "var(--red)", + segmentedActive: "var(--panel-3)", + + // Overlays + chrome + overlayBg: "rgba(2,2,6,0.94)", + editorBg: "var(--panel-2)", + shadowSm: "0 1px 2px rgba(0,0,0,0.18)", + shadowMdLight: "var(--shadow)", + shadowMdDark: "var(--shadow)", + shadowLgLight: "0 30px 70px -20px rgba(15,23,42,0.25)", + shadowLgDark: "0 30px 70px -20px rgba(0,0,0,0.7)", + + fontFamily: "var(--font)", + radiusMenu: 14, + sidebarRadius: 14, +}; diff --git a/src/lib/server/rate-limit.ts b/src/lib/server/rate-limit.ts new file mode 100644 index 0000000..012bc2b --- /dev/null +++ b/src/lib/server/rate-limit.ts @@ -0,0 +1,86 @@ +/** + * Tiny in-process fixed-window rate limiter for the Smart Gallery AI routes. + * + * --------------------------------------------------------------------------- + * SCOPE AND LIMITATIONS — READ BEFORE RELYING ON THIS + * --------------------------------------------------------------------------- + * State lives in a plain `Map` in THIS process's memory. That means: + * + * - PER-INSTANCE, NOT GLOBAL. With N app instances behind a load balancer a + * caller gets up to N x the configured budget. On serverless platforms each + * cold start begins with an empty map, so the effective limit is weaker + * still. + * - NOT DURABLE. A restart or redeploy clears every counter. + * - FIXED WINDOW, NOT SLIDING. A caller can burst `max` at the very end of one + * window and `max` again at the start of the next — up to 2x `max` across a + * window boundary. Acceptable here; the goal is to bound runaway cost, not + * to meter precisely. + * + * It exists because the routes it guards spend real money on GPU inference and + * shipping them with NO limit at all is worse than shipping an imperfect one. + * + * REPLACE WITH REDIS (or the platform's rate limiter) BEFORE RUNNING MORE THAN + * ONE INSTANCE. The `limit()` signature is deliberately narrow so a Redis-backed + * implementation can drop straight in — the only change needed is making it + * async at the call sites. + * + * This is a throttle, not an authorization check. See src/lib/server/session.ts. + */ + +interface Window { + /** Requests counted so far in the current window. */ + count: number; + /** Epoch ms at which the current window ends. */ + resetAt: number; +} + +const windows = new Map(); + +/** Drop expired entries so the map cannot grow without bound. */ +const SWEEP_INTERVAL_MS = 60_000; +let lastSweep = 0; + +function sweep(now: number): void { + if (now - lastSweep < SWEEP_INTERVAL_MS) return; + lastSweep = now; + for (const [key, w] of windows) { + if (w.resetAt <= now) windows.delete(key); + } +} + +export interface LimitResult { + ok: boolean; + /** Seconds until the window resets. Send as `Retry-After` when `ok` is false. */ + retryAfter: number; +} + +/** + * Count one request against `key` and report whether it is allowed. + * + * @param key Caller identity — use `rateLimitKey()` from ./session. + * @param max Requests allowed per window. + * @param windowMs Window length in ms. + */ +export function limit(key: string, max: number, windowMs: number): LimitResult { + const now = Date.now(); + sweep(now); + + const existing = windows.get(key); + if (!existing || existing.resetAt <= now) { + windows.set(key, { count: 1, resetAt: now + windowMs }); + return { ok: true, retryAfter: 0 }; + } + + if (existing.count >= max) { + return { ok: false, retryAfter: Math.max(1, Math.ceil((existing.resetAt - now) / 1000)) }; + } + + existing.count += 1; + return { ok: true, retryAfter: 0 }; +} + +/** Test/maintenance helper — clears all counters. */ +export function resetAllLimits(): void { + windows.clear(); + lastSweep = 0; +} diff --git a/src/lib/server/runpod/base64.ts b/src/lib/server/runpod/base64.ts new file mode 100644 index 0000000..57fda9a --- /dev/null +++ b/src/lib/server/runpod/base64.ts @@ -0,0 +1,86 @@ +/** + * base64 / data-URI helpers shared by the RunPod endpoint wrappers. + * Different endpoints name their image field differently and some prefix a + * `data:` URI — these helpers normalize both. + * + * PROVENANCE: a faithful port of + * `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/base64.ts`. + * The `pickOutputImage` normalization is intentionally identical so both apps + * tolerate the same set of endpoint response shapes. + * + * SERVER-ONLY. + */ + +/** "data:image/png;base64,XXXX" -> "XXXX" (leaves a bare base64 string untouched). */ +export function stripDataUri(s: string): string { + if (!s.startsWith("data:")) return s; + const i = s.indexOf(","); + return i === -1 ? s : s.slice(i + 1); +} + +/** Wrap a bare base64 string in a data: URI. */ +export function toDataUri(b64: string, mime: string): string { + return `data:${mime};base64,${stripDataUri(b64)}`; +} + +/** + * Pull the first image-like base64 string out of a RunPod endpoint's `output`, + * regardless of which field name it used. Handles the common shapes: + * "iVBOR..." (raw string) + * { image: "..." } / { image_png } / { image_base64 } + * { images: ["..."] } (array) + * { output: { image: "..." } } (nested) + */ +export function pickOutputImage(output: unknown): string { + const s = findImageString(output, false); + if (!s) throw new Error("RunPod output did not contain an image."); + return stripDataUri(s); +} + +/** Keys whose name implies the value IS the image — any non-empty string is accepted. */ +const IMAGE_KEYS = ["image_png", "image", "image_base64", "images"] as const; +/** Generic wrapper keys — a string here must actually look like image data. */ +const CONTAINER_KEYS = ["output", "result", "data"] as const; + +/** A base64 image payload is long; a status/id string ("success", "job-abc") is short. */ +function looksLikeImageData(s: string): boolean { + if (s.startsWith("data:image/")) return true; + return s.length >= 256 && /^[A-Za-z0-9+/=\s]+$/.test(s.slice(0, 256)); +} + +/** + * `strict` is true when we descended through a generic wrapper key (output/result/ + * data), where a bare string could be a status/id rather than an image — so it must + * pass `looksLikeImageData`. Under an explicit image key (or at top level) any + * non-empty string is taken as the image. + */ +function findImageString(value: unknown, strict: boolean, depth = 0): string | undefined { + if (depth > 5) return undefined; + if (typeof value === "string") { + if (!value) return undefined; + return !strict || looksLikeImageData(value) ? value : undefined; + } + if (Array.isArray(value)) { + for (const el of value) { + const s = findImageString(el, strict, depth + 1); + if (s) return s; + } + return undefined; + } + if (value && typeof value === "object") { + const o = value as Record; + for (const key of IMAGE_KEYS) { + if (key in o) { + const s = findImageString(o[key], false, depth + 1); + if (s) return s; + } + } + for (const key of CONTAINER_KEYS) { + if (key in o) { + const s = findImageString(o[key], true, depth + 1); + if (s) return s; + } + } + } + return undefined; +} diff --git a/src/lib/server/runpod/client.ts b/src/lib/server/runpod/client.ts new file mode 100644 index 0000000..e94fe12 --- /dev/null +++ b/src/lib/server/runpod/client.ts @@ -0,0 +1,134 @@ +/** + * Low-level RunPod transport. Every model runs as its own RunPod endpoint; this + * handles the common request envelope, bearer auth, and both response modes: + * + * - `/runsync` (preferred): the job runs synchronously and the body already + * contains `output` (or IS the output for a custom handler). + * - `/run`: returns `{ id, status }`; we poll `/status/{id}` until the job + * reaches a terminal state or the time budget (kept under the 60s serverless + * function cap) is exhausted. + * + * RUNPOD_API_KEY is read here so callers never handle the secret directly, and + * it is NEVER echoed into an error message. Upstream error bodies are truncated + * to 160 chars before being surfaced. + * + * PROVENANCE: a faithful port of + * `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/client.ts` (same 55s + * budget, same polling, same RunpodError.status mapping). + * + * SERVER-ONLY. + */ + +export class RunpodError extends Error { + status: number; + constructor(message: string, status = 502) { + super(message); + this.name = "RunpodError"; + this.status = status; + } +} + +export interface RunpodCallOpts { + /** Human label used in error messages, e.g. "sd-inpaint". */ + name: string; + /** Full endpoint URL from the per-model env var (…/runsync or …/run). */ + url: string; + input: Record; + /** Total budget for the whole call incl. polling. Default 55s (< the 60s cap). */ + timeoutMs?: number; + pollIntervalMs?: number; +} + +interface RunpodEnvelope { + id?: string; + status?: string; + output?: unknown; + error?: unknown; +} + +export async function runpodCall(opts: RunpodCallOpts): Promise { + const { name, url, input } = opts; + const timeoutMs = opts.timeoutMs ?? 55_000; + const pollIntervalMs = opts.pollIntervalMs ?? 1500; + + const key = process.env.RUNPOD_API_KEY; + if (!key) throw new RunpodError("RUNPOD_API_KEY is not set.", 500); + + const deadline = Date.now() + timeoutMs; + const authHeaders = { authorization: `Bearer ${key}` }; + + let res: Response; + try { + res = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", ...authHeaders }, + body: JSON.stringify({ input }), + signal: AbortSignal.timeout(timeoutMs), + cache: "no-store", + }); + } catch { + throw new RunpodError(`Could not reach RunPod (${name}).`, 502); + } + if (!res.ok) { + // Truncated on purpose — never surface a full upstream body to the client. + const detail = (await res.text().catch(() => "")).slice(0, 160); + throw new RunpodError(`RunPod ${name} error (${res.status}). ${detail}`.trim(), 502); + } + + const data = (await res.json().catch(() => null)) as RunpodEnvelope | null; + if (!data || typeof data !== "object") { + throw new RunpodError(`RunPod ${name} returned an invalid response.`, 502); + } + + // Terminal failure reported in a job envelope. + if (data.status === "FAILED" || data.status === "CANCELLED") { + throw new RunpodError(`RunPod ${name} job ${data.status.toLowerCase()}.`, 502); + } + // No job id → this is not an async envelope; the body itself is the output. + // Covers custom /runsync handlers that return their result directly, even when + // it carries a `status` field (e.g. "success"). + if (data.id === undefined) { + return (data.output !== undefined ? data.output : data) as TOut; + } + // Job envelope that already carries a completed/inline output. + if (data.output !== undefined && (data.status === undefined || data.status === "COMPLETED")) { + return data.output as TOut; + } + + // Async: poll /status/{id} until COMPLETED / FAILED / the time budget runs out. + // Each wait + fetch is clamped to the remaining budget so the whole call stays + // under `timeoutMs` (kept below the 60s function cap). + const statusUrl = url.replace(/\/run(sync)?(\/?)$/, `/status/${data.id}`); + for (;;) { + const remaining = deadline - Date.now(); + if (remaining < 500) break; // not enough budget for another round + await sleep(Math.min(pollIntervalMs, remaining)); + const left = deadline - Date.now(); + if (left <= 0) break; + let sres: Response; + try { + sres = await fetch(statusUrl, { + headers: authHeaders, + signal: AbortSignal.timeout(Math.min(10_000, Math.max(1000, left))), + cache: "no-store", + }); + } catch { + continue; // transient — keep polling until the deadline + } + if (!sres.ok) continue; + const sdata = (await sres.json().catch(() => null)) as RunpodEnvelope | null; + if (!sdata) continue; + if (sdata.status === "COMPLETED") return sdata.output as TOut; + if (sdata.status === "FAILED" || sdata.status === "CANCELLED") { + throw new RunpodError(`RunPod ${name} job ${sdata.status.toLowerCase()}.`, 502); + } + } + throw new RunpodError( + `RunPod ${name} timed out (raise the endpoint's speed or use /runsync).`, + 504, + ); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/lib/server/runpod/endpoints.ts b/src/lib/server/runpod/endpoints.ts new file mode 100644 index 0000000..c3cb827 --- /dev/null +++ b/src/lib/server/runpod/endpoints.ts @@ -0,0 +1,301 @@ +/** + * Typed wrappers for each RunPod endpoint behind the Smart Gallery. + * Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input` + * contract the model spec documents, and normalizes the response. + * + * PROVENANCE: a faithful port of + * `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/endpoints.ts` — + * identical `normalizeDetections` / `normalizeBox` logic and identical env var + * names, so an endpoint deployed for the SDK demo works here unchanged. + * + * SERVER-ONLY. Missing/invalid URLs throw a RunpodError(500) so an unconfigured + * op surfaces as a clear message rather than a crash. + */ + +import { stripDataUri, pickOutputImage } from "./base64"; +import { RunpodError, runpodCall } from "./client"; +import type { + Img2ImgReq, + InpaintReq, + RunpodDetection, + RunpodImageResult, + TiltResult, + TranscriptResult, +} from "./types"; + +function envNum(v: string | undefined, fallback: number): number { + // Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which + // would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`. + if (v == null || v.trim() === "") return fallback; + const n = Number(v); + return Number.isFinite(n) ? n : fallback; +} + +function endpointUrl(envVar: string): string { + const url = process.env[envVar]; + if (!url || !/^https?:\/\//i.test(url)) { + throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500); + } + return url; +} + +/** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */ +async function imageOp( + name: string, + envVar: string, + input: Record, +): Promise { + const output = await runpodCall({ name, url: endpointUrl(envVar), input }); + return { imageBase64: pickOutputImage(output), mimeType: "image/png" }; +} + +// --------------------------------------------------------------------------- +// Image endpoints +// --------------------------------------------------------------------------- + +/** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */ +export function rpRemoveBackground(imageB64: string, model?: string): Promise { + return imageOp("background-removal", "RUNPOD_BG_REMOVE_URL", { + task: "remove-bg", + image: imageB64, + ...(model ? { model_name: model } : {}), + }); +} + +/** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the + * op / restore pass) is authoritative — it is not overridden by any env default. */ +export function rpUpscale( + imageB64: string, + scale: 2 | 4, + faceEnhance = false, +): Promise { + return imageOp("upscale", "RUNPOD_UPSCALE_URL", { + task: "upscale", + image: imageB64, + scale, + face_enhance: faceEnhance, + }); +} + +/** #8 DDColor B&W → colorize. */ +export function rpColorize(imageB64: string, inputSize?: number): Promise { + return imageOp("colorize", "RUNPOD_COLORIZE_URL", { + image: imageB64, + ...(inputSize ? { input_size: inputSize } : {}), + }); +} + +/** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */ +export function rpInpaint(p: InpaintReq): Promise { + const input: Record = { + task: "inpaint", + image: p.imageB64, + mask: p.maskB64, + prompt: p.prompt, + strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8), + guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7), + num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35), + }; + const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT; + if (negative) input.negative_prompt = negative; + if (p.seed != null) input.seed = p.seed; + return imageOp("sd-inpaint", "RUNPOD_SD_INPAINT_URL", input); +} + +/** #10 SD 3.5 general prompt edit (img2img, no mask). */ +export function rpImg2Img(p: Img2ImgReq): Promise { + const input: Record = { + task: "img2img", + image: p.imageB64, + prompt: p.prompt, + strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6), + guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7), + num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35), + }; + const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT; + if (negative) input.negative_prompt = negative; + if (p.seed != null) input.seed = p.seed; + return imageOp("sd-img2img", "RUNPOD_SD_IMG2IMG_URL", input); +} + +// --------------------------------------------------------------------------- +// #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1) +// --------------------------------------------------------------------------- + +export async function rpDetect( + imageB64: string, + width: number, + height: number, +): Promise { + const output = await runpodCall({ + name: "yolo-detect", + url: endpointUrl("RUNPOD_YOLO_URL"), + input: { image: imageB64, task: "detect" }, + }); + return normalizeDetections(output, width, height); +} + +function extractDetectionArray(output: unknown): unknown[] { + if (Array.isArray(output)) return output; + if (output && typeof output === "object") { + const o = output as Record; + for (const key of ["detections", "predictions", "objects", "results", "boxes"]) { + if (Array.isArray(o[key])) return o[key] as unknown[]; + } + } + return []; +} + +/** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */ +function firstLabel(...vals: unknown[]): string | undefined { + for (const v of vals) { + if (typeof v === "string" && v.trim()) return v.trim(); + } + return undefined; +} + +function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] { + const out: RunpodDetection[] = []; + for (const raw of extractDetectionArray(output)) { + if (!raw || typeof raw !== "object") continue; + const o = raw as Record; + // Prefer a human-readable name (ultralytics tojson puts the string in `name` + // and a numeric index in `class`); fall back to class_. Using firstLabel + // (not `??`) also means an explicit empty-string label doesn't get kept + dropped. + const classId = o.class_id ?? (typeof o.class === "number" ? o.class : undefined); + const label = ( + firstLabel(o.label, o.name, o.class_name, typeof o.class === "string" ? o.class : undefined) ?? + (classId != null ? `class_${classId}` : "object") + ).toLowerCase(); + const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0; + const box = normalizeBox(o, width, height); + if (box && label) out.push({ label, confidence, box }); + } + return out; +} + +function num(v: unknown): number | null { + const n = Number(v); + return Number.isFinite(n) ? n : null; +} + +function asNum4(v: unknown): [number, number, number, number] | null { + if (!Array.isArray(v) || v.length < 4) return null; + const a = num(v[0]); + const b = num(v[1]); + const c = num(v[2]); + const d = num(v[3]); + return a === null || b === null || c === null || d === null ? null : [a, b, c, d]; +} + +/** + * Normalize a detection box to {x, y, width, height} as fractions 0..1 of the + * image, from whatever shape the endpoint emits: + * - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form + * - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form + * - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners + * - object `{x,y,width,height}` → x/y/width/height + * Pixel values (any component > 1) are divided by the image dims; already- + * normalized fractions pass through. + */ +function normalizeBox( + o: Record, + width: number, + height: number, +): RunpodDetection["box"] | null { + const W = width || 1; + const H = height || 1; + const clamp01 = (n: number) => Math.max(0, Math.min(1, n)); + const frac = (x: number, y: number, w: number, h: number): RunpodDetection["box"] => { + if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) { + x /= W; + y /= H; + w /= W; + h /= H; + } + return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) }; + }; + const fromXyxy = (x1: number, y1: number, x2: number, y2: number) => + frac(x1, y1, x2 - x1, y2 - y1); + + // 1. Array boxes, interpreted by which key holds them. + const xyxyArr = asNum4(o.xyxy); + if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]); + const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h] + if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]); + const boxArr = asNum4(o.box); // generic array box → assume corner form + if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]); + + // 2. Object boxes (either nested under `box` or directly on the detection). + const src = + o.box && typeof o.box === "object" && !Array.isArray(o.box) + ? (o.box as Record) + : o; + const x1 = num(src.x1 ?? src.left); + const y1 = num(src.y1 ?? src.top); + const x2 = num(src.x2 ?? src.right); + const y2 = num(src.y2 ?? src.bottom); + if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2); + + const x = num(src.x); + const y = num(src.y); + const w = num(src.width); + const h = num(src.height); + if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h); + + return null; +} + +// --------------------------------------------------------------------------- +// Audio / calibration endpoints +// --------------------------------------------------------------------------- + +/** #2 Camera tilt (DeepSingleImageCalibration). */ +export async function rpTilt(imageB64: string): Promise { + const o = await runpodCall>({ + name: "tilt", + url: endpointUrl("RUNPOD_TILT_URL"), + input: { image: imageB64 }, + }); + return { + rollDegrees: Number(o.roll_degrees ?? 0) || 0, + pitchDegrees: Number(o.pitch_degrees ?? 0) || 0, + fovDegrees: Number(o.fov_degrees ?? 0) || 0, + }; +} + +/** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */ +export async function rpTranscribe( + audioB64: string, + opts?: { language?: string; timestamps?: boolean; punctuation?: boolean }, +): Promise { + const o = await runpodCall>({ + name: "transcribe", + url: endpointUrl("RUNPOD_STT_URL"), + input: { audio: audioB64, task: "transcribe", ...(opts ?? {}) }, + }); + const rawSegments = Array.isArray(o.segments) ? o.segments : []; + const segments = rawSegments.map((s) => { + const seg = (s ?? {}) as Record; + return { + text: String(seg.text ?? ""), + startSec: Number(seg.start_sec ?? 0) || 0, + endSec: Number(seg.end_sec ?? 0) || 0, + }; + }); + return { + transcript: String(o.transcript ?? ""), + segments: segments.length ? segments : undefined, + }; +} + +/** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */ +export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> { + const o = await runpodCall>({ + name: "audio-denoise", + url: endpointUrl("RUNPOD_AUDIO_DENOISE_URL"), + input: { audio: audioB64, task: "denoise" }, + }); + const a = o.audio ?? o.output ?? ""; + return { audioB64: stripDataUri(String(a)) }; +} diff --git a/src/lib/server/runpod/types.ts b/src/lib/server/runpod/types.ts new file mode 100644 index 0000000..19313e9 --- /dev/null +++ b/src/lib/server/runpod/types.ts @@ -0,0 +1,68 @@ +/** + * Request/response types for the RunPod endpoints behind the Smart Gallery. + * + * PROVENANCE: a faithful port of + * `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/types.ts`. + * Keep the two in sync — the RunPod endpoints are shared between the standalone + * SDK demo and this CRM. + * + * SERVER-ONLY. Imported by src/app/api/gallery/ai/* route handlers, never by a + * client component (which must not see a RunPod URL or key). + */ + +/** Normalized image result returned by every image endpoint (raw base64, no data: prefix). */ +export interface RunpodImageResult { + imageBase64: string; + mimeType: string; +} + +/** #9 SD 3.5 masked inpainting (also #11 outpaint, pre-padded). white in mask = regenerate. */ +export interface InpaintReq { + imageB64: string; + maskB64: string; + prompt: string; + negativePrompt?: string; + strength?: number; + guidanceScale?: number; + steps?: number; + seed?: number; +} + +/** #10 SD 3.5 general prompt edit (img2img, no mask). */ +export interface Img2ImgReq { + imageB64: string; + prompt: string; + negativePrompt?: string; + strength?: number; + guidanceScale?: number; + steps?: number; + seed?: number; +} + +/** + * #1 YOLO construction-material classifier, normalized to the SDK's + * `DetectedObject` shape (box as fractions 0..1 of the image). + */ +export interface RunpodDetection { + label: string; + confidence: number; + box: { x: number; y: number; width: number; height: number }; +} + +/** #2 DeepSingleImageCalibration — camera tilt. */ +export interface TiltResult { + rollDegrees: number; + pitchDegrees: number; + fovDegrees: number; +} + +/** #3 Parakeet voice-to-text. */ +export interface TranscriptSegment { + text: string; + startSec: number; + endSec: number; +} +export interface TranscriptResult { + transcript: string; + segments?: TranscriptSegment[]; +} diff --git a/src/lib/server/session.ts b/src/lib/server/session.ts new file mode 100644 index 0000000..46de022 --- /dev/null +++ b/src/lib/server/session.ts @@ -0,0 +1,139 @@ +/** + * Session gate for the Smart Gallery AI routes. + * + * --------------------------------------------------------------------------- + * TRUST MODEL + * --------------------------------------------------------------------------- + * These routes proxy a paid, rate-limited GPU backend (RunPod) using a secret + * held only on this server. An unauthenticated route is therefore not merely an + * information-disclosure problem — it is a billable-resource problem: anyone who + * can reach the URL can spend the operator's GPU budget, and can use the CRM as + * an open relay for arbitrary image/audio processing. + * + * The upstream SDK demo's routes (apps/web/src/app/api/ai/*) are COMPLETELY + * unauthenticated. This module is the fix; every ported route must call it + * before doing any work. + * + * WHO IS TRUSTED + * We do not verify a JWT here and we do not hold any signing key. The single + * source of truth for "is this caller signed in" is the Shell BFF + * (`${BFF_ORIGIN}/api/session/context`), which owns the HttpOnly session cookie. + * We forward the caller's raw `cookie` header to it and treat a 200 as proof of + * a session. Consequences of that choice, stated explicitly: + * + * - The BFF is trusted absolutely. If it is compromised or misconfigured to + * answer 200 for anonymous callers, these routes are open. BFF_ORIGIN must + * therefore only ever point at an origin the operator controls. + * - We forward the cookie header verbatim and nothing else. No Authorization + * header, no bearer token, and never the RunPod key. + * - NOTHING IS CACHED. A cached "yes" would keep a revoked/expired session + * alive for the cache lifetime, so every AI request costs one BFF round + * trip. That is deliberate: correctness over latency for a spend gate. + * - A network failure reaching the BFF returns 503 (fail CLOSED), never 200. + * If we cannot prove a session, we do not spend GPU budget. + * + * DEMO MODE + * When the Shell is not configured (`NEXT_PUBLIC_SUPABASE_URL` unset) the CRM + * runs on its mock portal and there is no session to check, so we allow the + * request and log ONCE at startup-of-first-use. This is the same gate + * `isShellConfigured()` uses for mock-vs-real auth elsewhere in the app. + * IMPORTANT: never deploy to a public origin with the Shell unconfigured AND a + * real RUNPOD_API_KEY present — that combination is an open, billable endpoint. + * + * WHAT THIS IS NOT + * This is authentication only, not authorization. It answers "is there a valid + * session", not "may this principal use the gallery". Per-resource policy for + * gallery data lives in be-crm behind the `crm.gallery` resource; if these AI + * routes ever need the same, check it there rather than re-deriving it here. + */ + +export type GallerySession = + | { ok: true; principalId?: string } + | { ok: false; status: number; error: string }; + +/** Mirrors src/lib/appshell.ts — kept local so this stays server-only. */ +function isShellConfigured(): boolean { + return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL); +} + +let demoModeWarned = false; + +/** + * Resolve the caller's session. Returns `{ ok: true }` (optionally with the + * principal id, used to key rate limits) or a ready-to-return failure with the + * status and message the route should emit. + */ +export async function requireGallerySession(req: Request): Promise { + if (!isShellConfigured()) { + if (!demoModeWarned) { + demoModeWarned = true; + console.warn( + "[gallery-ai] Shell is not configured (NEXT_PUBLIC_SUPABASE_URL unset) — " + + "AI routes are UNAUTHENTICATED in demo mode. Do not expose this deployment publicly " + + "while RUNPOD_API_KEY is set.", + ); + } + return { ok: true }; + } + + const cookie = req.headers.get("cookie"); + if (!cookie) return { ok: false, status: 401, error: "Not signed in" }; + + const origin = (process.env.BFF_ORIGIN ?? "http://localhost:4000").replace(/\/$/, ""); + + let res: Response; + try { + res = await fetch(`${origin}/api/session/context`, { + headers: { cookie, accept: "application/json" }, + // Never cache an auth decision — see the trust-model note above. + cache: "no-store", + signal: AbortSignal.timeout(5000), + }); + } catch { + // Fail closed: we could not prove a session, so we do not spend GPU budget. + return { ok: false, status: 503, error: "Session service unavailable" }; + } + + if (res.status === 401) return { ok: false, status: 401, error: "Not signed in" }; + if (!res.ok) return { ok: false, status: 503, error: "Session service unavailable" }; + + // The principal id is best-effort: it only sharpens the rate-limit key, so a + // shape we don't recognize degrades to IP-keyed limiting rather than failing. + let principalId: string | undefined; + try { + const data = (await res.json()) as Record | null; + principalId = pickPrincipalId(data); + } catch { + /* ignore — see above */ + } + + return principalId ? { ok: true, principalId } : { ok: true }; +} + +function pickPrincipalId(data: Record | null): string | undefined { + if (!data) return undefined; + const direct = data.userId ?? data.principalId ?? data.sub ?? data.id; + if (typeof direct === "string" && direct) return direct; + const user = data.user; + if (user && typeof user === "object") { + const u = user as Record; + const nested = u.id ?? u.userId ?? u.sub; + if (typeof nested === "string" && nested) return nested; + } + return undefined; +} + +/** + * Rate-limit key for a request: the authenticated principal when known, + * otherwise the first hop of `x-forwarded-for`. + * + * NOTE the first hop is client-controlled unless a trusted proxy overwrites the + * header. It is good enough to throttle honest clients and casual abuse; it is + * NOT a security boundary. The session gate above is the security boundary. + */ +export function rateLimitKey(req: Request, principalId?: string): string { + if (principalId) return `u:${principalId}`; + const xff = req.headers.get("x-forwarded-for") ?? ""; + const first = xff.split(",")[0]?.trim(); + return `ip:${first || "unknown"}`; +} diff --git a/vendor/README.md b/vendor/README.md new file mode 100644 index 0000000..beecccd --- /dev/null +++ b/vendor/README.md @@ -0,0 +1,38 @@ +# vendor/ + +Third-party source vendored into this repo so the app builds from a **single checkout**. + +## `photo-gallery-sdk/` + +`@photo-gallery/sdk` — the Smart Gallery engine. Upstream: +`https://git.lynkedup.cloud/KaushikRK99/advance-photo-gallery-web-sdk` (`packages/photo-sdk`). + +**This directory is generated. Do not hand-edit it** — changes belong upstream, then: + +```bash +npm run sync:gallery-sdk # re-copies src/ + package.json + README.md from the sibling checkout +git add vendor/photo-gallery-sdk && git commit -m "chore: sync @photo-gallery/sdk" +``` + +### Why vendored instead of a registry dependency + +The SDK is not published anywhere yet. A `file:../advance-photo-gallery-web-sdk/...` dependency +works on a developer machine that has both repos side by side, but **fails every CI/Vercel build**, +because the deploy only ever checks out this repo. Vendoring keeps the dependency graph closed. + +Only `src/`, `package.json` and `README.md` are copied — the package's `exports` point at `src/` +and `next.config.ts` lists it in `transpilePackages`, so no build output is needed. Its own +`node_modules` is deliberately excluded: without it, the SDK's sources resolve `@types/react` from +this repo's React 19 rather than the pnpm workspace's React 18, which is what lets them type-check +here at all. + +### Replacing this with a real dependency + +When the SDK is published to the Gitea npm registry (the same one `@insignia/*` uses — Vercel +already has `GITEA_TOKEN` for it): + +1. Publish it from the SDK repo under a scope this repo's `.npmrc` maps to the registry. +2. In `package.json`, swap `"@photo-gallery/sdk": "file:./vendor/photo-gallery-sdk"` for the version range. +3. Delete this directory and the `sync:gallery-sdk` script. + +Nothing else changes — no import paths, no config. diff --git a/vendor/photo-gallery-sdk/README.md b/vendor/photo-gallery-sdk/README.md new file mode 100644 index 0000000..48dfd43 --- /dev/null +++ b/vendor/photo-gallery-sdk/README.md @@ -0,0 +1,288 @@ +# @photo-gallery/sdk + +A reusable, macOS Photos-style photo gallery for React / Next.js — light & dark, fully responsive, +pluggable storage and AI. One component, no required CSS framework. + +```tsx +import { PhotoGallery } from '@photo-gallery/sdk'; +import '@photo-gallery/sdk/styles.css'; + +export default () => ; +``` + +See the [repository README](../../README.md) for full documentation, props, adapters and the +AI-provider interface. + +## Embedding in a host app + +By default `` behaves like a standalone app: it owns the sidebar, the toolbar, the +theme switcher and the global keyboard shortcuts, and it assumes a full-viewport parent. All of +that is opt-out, so the gallery can be dropped into an existing product's shell instead. + +```tsx +
+ +
+``` + +### New props + +| Prop | Type | Default | What it does | +| --- | --- | --- | --- | +| `embedded` | `boolean` | `false` | Lays out at `height: 100%` inside the host container instead of assuming a full-viewport parent, and switches interactive elements to `cursor: pointer`. | +| `currentUser` | `GalleryUser` | – | The host's signed-in user (`{ id, name, email?, avatarUrl? }`). Comments are stamped with this identity; the free-text author field disappears. | +| `shareBaseUrl` | `string` | `${location.origin}/gallery` | Base URL for generated share links (`?shared=` is appended). | +| `chrome` | `Partial` | `DEFAULT_CHROME` | Suppress chrome the host already provides: `{ titlebar, sidebar, toolbar, themeSwitcher }`. | +| `hiddenViews` | `ViewId[]` | `[]` | Sidebar rows + Collections cards to hide, e.g. `['screenshots', 'sys:documents']`. A section label drops when all its rows are hidden; if the active view becomes hidden the gallery falls back to `library`. | +| `keyboardShortcuts` | `boolean` | `true` | Bind global shortcuts (⌘A, Delete, F, Esc, Enter) to `window`. | +| `defaultFullscreen` | `boolean` | `false` | Start maximised over the host page. | +| `lockProvider` | `LockProvider` | – | Server-backed, per-user lock for Recently Deleted. Replaces the device-local localStorage hash entirely. | + +`showWindowChrome` still works and maps onto `chrome.titlebar`; an explicit `chrome.titlebar` wins. +`DEFAULT_CHROME` is exported (`{ titlebar: false, sidebar: true, toolbar: true, themeSwitcher: true }`). + +Turning `chrome.themeSwitcher` off removes the Appearance items from the "More" menu, so a host that +owns light/dark cannot be overridden from inside the gallery. The config is now **live**: changing +`theme`, `themeTokens`, `currentUser` or `chrome` re-applies without remounting the gallery. + +### Full screen / maximise + +The toolbar carries a maximise/restore button (right-hand group, next to Info). Turning it on adds +`apg--fullscreen` to the root element, which goes `position: fixed; inset: 0; z-index: 1400; +height: 100dvh; border-radius: 0` — so the gallery fills the viewport no matter what height the host +container gave it, and sits above the host's own chrome. Escape leaves full screen, but only once +nothing else owns Escape: the lightbox, the photo/video editors, the camera, modals and context menus +all get it first, and it still clears an object focus or a selection before it un-maximises. Escape is +part of the shortcut set, so `keyboardShortcuts={false}` opts out of that too — the button always works. + +The SDK's own overlays needed no change. They are `position: fixed; inset: 0`, which resolves against +the viewport — and in full screen the gallery covers exactly the viewport, so they still land right. +`z-index: 1400` opens a stacking context, so their existing 1000–1300 z-indexes now stack *inside* it: +above the gallery's content and, through 1400, above the host's chrome. + +Headless control lives on the store: `fullscreen`, `setFullscreen(next)`, `toggleFullscreen()`. +`defaultFullscreen` only seeds the initial value; the user's toggle owns it afterwards. + +### Server-backed Recently Deleted lock + +By default the Recently Deleted lock is a password hash in `localStorage` — device-local, and invisible +to your backend. Pass a `lockProvider` and the SDK delegates every lock operation to it instead, so the +lock belongs to the *user* and follows them across devices. The localStorage path is untouched when no +provider is given. + +```tsx + call('crm.gallery.lock.status', {}), + set: async (password) => { await call('crm.gallery.lock.set', { password }); }, + verify: async (password) => (await call('crm.gallery.lock.verify', { password })).ok, + }} +/> +``` + +```ts +interface LockProvider { + status(): Promise<{ hasPassword: boolean }>; + set(password: string | null): Promise; // null clears it + verify(password: string): Promise; +} +``` + +Drive your UI off `lockConfigured: boolean` (refreshed from `status()` during `init()`, and again if +the provider arrives late), not `lock.hash` — that field is meaningless on the provider path. Failures +are distinguished: a `verify()` that *resolves false* sets `lockError: 'wrong-password'` ("Incorrect +password."), while one that *rejects* sets `lockError: 'unavailable'` ("Couldn't check the password. +Please try again.") so the user retries instead of doubting what they typed. `clearLockError()` resets +it; `refreshLockStatus()` re-reads `status()` on demand. + +### Theme tokens → CSS variables + +Every value is optional. Tokens ending in `Light`/`Dark` are theme-paired (the matching one is +applied for the resolved theme); bare names are theme-independent. Numbers are emitted as `px`. +Values are rejected if they contain `url(`, `expression(`, `javascript:` or `<>{}`. + +| Token | CSS variable | Notes | +| --- | --- | --- | +| `accent` | `--apg-accent` | Also overrides the `accentColor` prop. | +| `accentStrongLight` / `accentStrongDark` | `--apg-accent-strong` | Accent behind white text (AA contrast). | +| `accentContrast` | `--apg-accent-contrast` | Foreground on top of the accent. | +| `dangerLight` / `dangerDark` | `--apg-danger` | Destructive actions. | +| `bgLight` / `bgDark` | `--apg-bg` + `--apg-bg-content` | App / content background. | +| `elevatedLight` / `elevatedDark` | `--apg-bg-elevated` | Raised surfaces. | +| `cardLight` / `cardDark` | `--apg-card` | Collection cards. | +| `cardHoverLight` / `cardHoverDark` | `--apg-card-hover` | Card hover state. | +| `sidebarBgLight` / `sidebarBgDark` | `--apg-sidebar-bg` | Sidebar glass (semi-dark always uses the dark value). | +| `toolbarBgLight` / `toolbarBgDark` | `--apg-toolbar-bg` | Top toolbar glass. | +| `menuBgLight` / `menuBgDark` | `--apg-menu-bg` | Context menus, Info panel, popovers. | +| `separatorLight` / `separatorDark` | `--apg-separator` | Hairlines. | +| `separatorStrongLight` / `separatorStrongDark` | `--apg-separator-strong` | Input borders, scrollbars. | +| `hoverLight` / `hoverDark` | `--apg-hover` | Row / icon-button hover wash. | +| `activeLight` / `activeDark` | `--apg-active` | Pressed state. | +| `sidebarSelectedLight` / `sidebarSelectedDark` | `--apg-sidebar-selected` | Selected sidebar row. | +| `textLight` / `textDark` | `--apg-text` | Primary text. | +| `textSecondaryLight` / `textSecondaryDark` | `--apg-text-secondary` | Labels, captions. | +| `textTertiaryLight` / `textTertiaryDark` | `--apg-text-tertiary` | Hints, timestamps. | +| `glassBorderLight` / `glassBorderDark` | `--apg-glass-border` | Border on glass surfaces. | +| `fontFamily` | `--apg-font` | Font stack for the whole gallery. | +| `sidebarRadius` | `--apg-sidebar-radius` | px. | +| `radiusMenu` | `--apg-radius-menu` | px. | +| `shadowSm` | `--apg-shadow-sm` | | +| `shadowMdLight` / `shadowMdDark` | `--apg-shadow-md` | Menus, action bar. | +| `shadowLgLight` / `shadowLgDark` | `--apg-shadow-lg` | Modals. | +| `tileFav` | `--apg-tile-fav` | Favourite heart on a tile (default `#ff3b30`). | +| `overlayBg` | `--apg-overlay-bg` | Lightbox backdrop (default `rgba(0,0,0,0.97)`). | +| `editorBg` | `--apg-editor-bg` | Editor chrome (default `#161617`). | +| `segmentedActive` | `--apg-segmented-active` | Active segmented pill. | +| `sidebarWidth` | `--apg-sidebar-w` | px. | +| `toolbarHeight` | `--apg-toolbar-h` | px. Also drives `--apg-overlay-top`. | + +The Info panel is positioned at `top: var(--apg-overlay-top, 64px)` — set `--apg-overlay-top` via the +`style` prop to push it below the host's own header. Full-screen overlays (lightbox, editor, camera, +modals, context menus) stay `position: fixed`, which is correct for a modal over a host app. + +### Incremental storage adapters + +`StorageAdapter` gained two optional members. Both are additive — existing `save`/`putBlob` adapters +keep working unchanged. + +- **`applyChanges(changes: StateChanges)`** — when present the store calls this *instead of* `save()` + on every (debounced, 400 ms) change, sending only what differs from the last persisted snapshot: + `{ upsertMedia?, removeMedia?, upsertAlbums?, removeAlbums?, upsertPeople?, removePeople?, + labelAliases?, deletedLabels? }`. Overlapping writes are serialized, and a rejection rolls the + snapshot back so the next persist retries the same diff. Only non-system albums are ever persisted. +- **`putMedia(id, blob, { name, mime }): Promise`** — preferred over `putBlob`. It returns + `{ ref, url }`: the store sets `item.src = url` (renderable now, may be a short-lived signed URL) + and `item.storageRef = ref` (the durable reference that survives a reload). + +Worked example against a data-door backend: + +```ts +import type { StorageAdapter, StateChanges, StoredBlob } from '@photo-gallery/sdk'; + +export function createDataDoorAdapter(call: (a: string, p: unknown) => Promise): StorageAdapter { + return { + name: 'data-door', + + async load() { + return await call('crm.gallery.state.load', {}); + }, + + // Kept as a fallback for callers that don't use applyChanges. + async save(state) { + await call('crm.gallery.state.apply', { + upsertMedia: state.media, + upsertAlbums: state.albums, + upsertPeople: state.people, + labelAliases: state.labelAliases, + deletedLabels: state.deletedLabels, + } satisfies StateChanges); + }, + + async applyChanges(changes: StateChanges) { + await call('crm.gallery.state.apply', changes); + }, + + async putMedia(id, blob, meta): Promise { + const { ref, uploadUrl, method, headers } = await call('crm.gallery.media.presignUpload', { + mediaId: id, + mime: meta.mime, + sizeBytes: blob.size, + filename: meta.name, + }); + await fetch(uploadUrl, { method, headers, body: blob }); + const { urls } = await call('crm.gallery.media.presignDownload', { refs: [ref] }); + return { ref, url: urls[ref] }; + }, + }; +} +``` + +### Identity-aware comments + +With `currentUser` set, `addComment(id, text)` stamps `{ authorId, author, authorAvatar }` from the +configured user, the Info panel renders the real avatar/name instead of a name input (and stops using +the `apg:comment-author` localStorage key), and delete affordances appear only on comments whose +`authorId` matches. `deleteComment` is a client-side no-op for someone else's comment — the backend +remains the authority. Without `currentUser`, behaviour is exactly as before. + +### Uploader & version identity + +With `currentUser` set, imports (`importFiles`) and camera captures stamp `MediaItem.uploadedBy` +(a `GalleryUser`) — a re-import never overwrites an existing value. Saving an edit (`addVersion`) or +restoring one (`restoreVersion`) stamps `{ authorId, author, authorAvatar }` onto the `MediaVersion`. +The Info panel shows an "Uploaded by" block (avatar + name + muted email) and the author beside each +version's timestamp; the Versions & Audit view shows the latest editor. All fields are optional — with +no `currentUser`, nothing is stamped and the UI renders exactly as before. `be-crm` still owns +`owner_principal_id`; `uploadedBy` is display identity only. + +### Editable caption & notes + +The Info panel's caption and a new multi-line `MediaItem.note` are click-to-edit: they save on blur or +Enter (⌘/Ctrl+Enter for the note) via `updateMedia`, so they persist through the adapter. `note` is +folded into the search haystack, so notes are searchable. + +### Info panel media & comments + +Video items render a real inline `