feat: add core domain types for Photo Gallery SDK including media items, albums, and annotations
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* Tiny in-process fixed-window rate limiter for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* SCOPE AND LIMITATIONS — READ BEFORE RELYING ON THIS
|
||||
* ---------------------------------------------------------------------------
|
||||
* State lives in a plain `Map` in THIS process's memory. That means:
|
||||
*
|
||||
* - PER-INSTANCE, NOT GLOBAL. With N app instances behind a load balancer a
|
||||
* caller gets up to N x the configured budget. On serverless platforms each
|
||||
* cold start begins with an empty map, so the effective limit is weaker
|
||||
* still.
|
||||
* - NOT DURABLE. A restart or redeploy clears every counter.
|
||||
* - FIXED WINDOW, NOT SLIDING. A caller can burst `max` at the very end of one
|
||||
* window and `max` again at the start of the next — up to 2x `max` across a
|
||||
* window boundary. Acceptable here; the goal is to bound runaway cost, not
|
||||
* to meter precisely.
|
||||
*
|
||||
* It exists because the routes it guards spend real money on GPU inference and
|
||||
* shipping them with NO limit at all is worse than shipping an imperfect one.
|
||||
*
|
||||
* REPLACE WITH REDIS (or the platform's rate limiter) BEFORE RUNNING MORE THAN
|
||||
* ONE INSTANCE. The `limit()` signature is deliberately narrow so a Redis-backed
|
||||
* implementation can drop straight in — the only change needed is making it
|
||||
* async at the call sites.
|
||||
*
|
||||
* This is a throttle, not an authorization check. See src/lib/server/session.ts.
|
||||
*/
|
||||
|
||||
interface Window {
|
||||
/** Requests counted so far in the current window. */
|
||||
count: number;
|
||||
/** Epoch ms at which the current window ends. */
|
||||
resetAt: number;
|
||||
}
|
||||
|
||||
const windows = new Map<string, Window>();
|
||||
|
||||
/** Drop expired entries so the map cannot grow without bound. */
|
||||
const SWEEP_INTERVAL_MS = 60_000;
|
||||
let lastSweep = 0;
|
||||
|
||||
function sweep(now: number): void {
|
||||
if (now - lastSweep < SWEEP_INTERVAL_MS) return;
|
||||
lastSweep = now;
|
||||
for (const [key, w] of windows) {
|
||||
if (w.resetAt <= now) windows.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
export interface LimitResult {
|
||||
ok: boolean;
|
||||
/** Seconds until the window resets. Send as `Retry-After` when `ok` is false. */
|
||||
retryAfter: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count one request against `key` and report whether it is allowed.
|
||||
*
|
||||
* @param key Caller identity — use `rateLimitKey()` from ./session.
|
||||
* @param max Requests allowed per window.
|
||||
* @param windowMs Window length in ms.
|
||||
*/
|
||||
export function limit(key: string, max: number, windowMs: number): LimitResult {
|
||||
const now = Date.now();
|
||||
sweep(now);
|
||||
|
||||
const existing = windows.get(key);
|
||||
if (!existing || existing.resetAt <= now) {
|
||||
windows.set(key, { count: 1, resetAt: now + windowMs });
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
if (existing.count >= max) {
|
||||
return { ok: false, retryAfter: Math.max(1, Math.ceil((existing.resetAt - now) / 1000)) };
|
||||
}
|
||||
|
||||
existing.count += 1;
|
||||
return { ok: true, retryAfter: 0 };
|
||||
}
|
||||
|
||||
/** Test/maintenance helper — clears all counters. */
|
||||
export function resetAllLimits(): void {
|
||||
windows.clear();
|
||||
lastSweep = 0;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* base64 / data-URI helpers shared by the RunPod endpoint wrappers.
|
||||
* Different endpoints name their image field differently and some prefix a
|
||||
* `data:` URI — these helpers normalize both.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/base64.ts`.
|
||||
* The `pickOutputImage` normalization is intentionally identical so both apps
|
||||
* tolerate the same set of endpoint response shapes.
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
/** "data:image/png;base64,XXXX" -> "XXXX" (leaves a bare base64 string untouched). */
|
||||
export function stripDataUri(s: string): string {
|
||||
if (!s.startsWith("data:")) return s;
|
||||
const i = s.indexOf(",");
|
||||
return i === -1 ? s : s.slice(i + 1);
|
||||
}
|
||||
|
||||
/** Wrap a bare base64 string in a data: URI. */
|
||||
export function toDataUri(b64: string, mime: string): string {
|
||||
return `data:${mime};base64,${stripDataUri(b64)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the first image-like base64 string out of a RunPod endpoint's `output`,
|
||||
* regardless of which field name it used. Handles the common shapes:
|
||||
* "iVBOR..." (raw string)
|
||||
* { image: "..." } / { image_png } / { image_base64 }
|
||||
* { images: ["..."] } (array)
|
||||
* { output: { image: "..." } } (nested)
|
||||
*/
|
||||
export function pickOutputImage(output: unknown): string {
|
||||
const s = findImageString(output, false);
|
||||
if (!s) throw new Error("RunPod output did not contain an image.");
|
||||
return stripDataUri(s);
|
||||
}
|
||||
|
||||
/** Keys whose name implies the value IS the image — any non-empty string is accepted. */
|
||||
const IMAGE_KEYS = ["image_png", "image", "image_base64", "images"] as const;
|
||||
/** Generic wrapper keys — a string here must actually look like image data. */
|
||||
const CONTAINER_KEYS = ["output", "result", "data"] as const;
|
||||
|
||||
/** A base64 image payload is long; a status/id string ("success", "job-abc") is short. */
|
||||
function looksLikeImageData(s: string): boolean {
|
||||
if (s.startsWith("data:image/")) return true;
|
||||
return s.length >= 256 && /^[A-Za-z0-9+/=\s]+$/.test(s.slice(0, 256));
|
||||
}
|
||||
|
||||
/**
|
||||
* `strict` is true when we descended through a generic wrapper key (output/result/
|
||||
* data), where a bare string could be a status/id rather than an image — so it must
|
||||
* pass `looksLikeImageData`. Under an explicit image key (or at top level) any
|
||||
* non-empty string is taken as the image.
|
||||
*/
|
||||
function findImageString(value: unknown, strict: boolean, depth = 0): string | undefined {
|
||||
if (depth > 5) return undefined;
|
||||
if (typeof value === "string") {
|
||||
if (!value) return undefined;
|
||||
return !strict || looksLikeImageData(value) ? value : undefined;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
for (const el of value) {
|
||||
const s = findImageString(el, strict, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
const o = value as Record<string, unknown>;
|
||||
for (const key of IMAGE_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], false, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
for (const key of CONTAINER_KEYS) {
|
||||
if (key in o) {
|
||||
const s = findImageString(o[key], true, depth + 1);
|
||||
if (s) return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
/**
|
||||
* Low-level RunPod transport. Every model runs as its own RunPod endpoint; this
|
||||
* handles the common request envelope, bearer auth, and both response modes:
|
||||
*
|
||||
* - `/runsync` (preferred): the job runs synchronously and the body already
|
||||
* contains `output` (or IS the output for a custom handler).
|
||||
* - `/run`: returns `{ id, status }`; we poll `/status/{id}` until the job
|
||||
* reaches a terminal state or the time budget (kept under the 60s serverless
|
||||
* function cap) is exhausted.
|
||||
*
|
||||
* RUNPOD_API_KEY is read here so callers never handle the secret directly, and
|
||||
* it is NEVER echoed into an error message. Upstream error bodies are truncated
|
||||
* to 160 chars before being surfaced.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/client.ts` (same 55s
|
||||
* budget, same polling, same RunpodError.status mapping).
|
||||
*
|
||||
* SERVER-ONLY.
|
||||
*/
|
||||
|
||||
export class RunpodError extends Error {
|
||||
status: number;
|
||||
constructor(message: string, status = 502) {
|
||||
super(message);
|
||||
this.name = "RunpodError";
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export interface RunpodCallOpts {
|
||||
/** Human label used in error messages, e.g. "sd-inpaint". */
|
||||
name: string;
|
||||
/** Full endpoint URL from the per-model env var (…/runsync or …/run). */
|
||||
url: string;
|
||||
input: Record<string, unknown>;
|
||||
/** Total budget for the whole call incl. polling. Default 55s (< the 60s cap). */
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
}
|
||||
|
||||
interface RunpodEnvelope {
|
||||
id?: string;
|
||||
status?: string;
|
||||
output?: unknown;
|
||||
error?: unknown;
|
||||
}
|
||||
|
||||
export async function runpodCall<TOut = unknown>(opts: RunpodCallOpts): Promise<TOut> {
|
||||
const { name, url, input } = opts;
|
||||
const timeoutMs = opts.timeoutMs ?? 55_000;
|
||||
const pollIntervalMs = opts.pollIntervalMs ?? 1500;
|
||||
|
||||
const key = process.env.RUNPOD_API_KEY;
|
||||
if (!key) throw new RunpodError("RUNPOD_API_KEY is not set.", 500);
|
||||
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const authHeaders = { authorization: `Bearer ${key}` };
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", ...authHeaders },
|
||||
body: JSON.stringify({ input }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
throw new RunpodError(`Could not reach RunPod (${name}).`, 502);
|
||||
}
|
||||
if (!res.ok) {
|
||||
// Truncated on purpose — never surface a full upstream body to the client.
|
||||
const detail = (await res.text().catch(() => "")).slice(0, 160);
|
||||
throw new RunpodError(`RunPod ${name} error (${res.status}). ${detail}`.trim(), 502);
|
||||
}
|
||||
|
||||
const data = (await res.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!data || typeof data !== "object") {
|
||||
throw new RunpodError(`RunPod ${name} returned an invalid response.`, 502);
|
||||
}
|
||||
|
||||
// Terminal failure reported in a job envelope.
|
||||
if (data.status === "FAILED" || data.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${data.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
// No job id → this is not an async envelope; the body itself is the output.
|
||||
// Covers custom /runsync handlers that return their result directly, even when
|
||||
// it carries a `status` field (e.g. "success").
|
||||
if (data.id === undefined) {
|
||||
return (data.output !== undefined ? data.output : data) as TOut;
|
||||
}
|
||||
// Job envelope that already carries a completed/inline output.
|
||||
if (data.output !== undefined && (data.status === undefined || data.status === "COMPLETED")) {
|
||||
return data.output as TOut;
|
||||
}
|
||||
|
||||
// Async: poll /status/{id} until COMPLETED / FAILED / the time budget runs out.
|
||||
// Each wait + fetch is clamped to the remaining budget so the whole call stays
|
||||
// under `timeoutMs` (kept below the 60s function cap).
|
||||
const statusUrl = url.replace(/\/run(sync)?(\/?)$/, `/status/${data.id}`);
|
||||
for (;;) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining < 500) break; // not enough budget for another round
|
||||
await sleep(Math.min(pollIntervalMs, remaining));
|
||||
const left = deadline - Date.now();
|
||||
if (left <= 0) break;
|
||||
let sres: Response;
|
||||
try {
|
||||
sres = await fetch(statusUrl, {
|
||||
headers: authHeaders,
|
||||
signal: AbortSignal.timeout(Math.min(10_000, Math.max(1000, left))),
|
||||
cache: "no-store",
|
||||
});
|
||||
} catch {
|
||||
continue; // transient — keep polling until the deadline
|
||||
}
|
||||
if (!sres.ok) continue;
|
||||
const sdata = (await sres.json().catch(() => null)) as RunpodEnvelope | null;
|
||||
if (!sdata) continue;
|
||||
if (sdata.status === "COMPLETED") return sdata.output as TOut;
|
||||
if (sdata.status === "FAILED" || sdata.status === "CANCELLED") {
|
||||
throw new RunpodError(`RunPod ${name} job ${sdata.status.toLowerCase()}.`, 502);
|
||||
}
|
||||
}
|
||||
throw new RunpodError(
|
||||
`RunPod ${name} timed out (raise the endpoint's speed or use /runsync).`,
|
||||
504,
|
||||
);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
/**
|
||||
* Typed wrappers for each RunPod endpoint behind the Smart Gallery.
|
||||
* Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input`
|
||||
* contract the model spec documents, and normalizes the response.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/endpoints.ts` —
|
||||
* identical `normalizeDetections` / `normalizeBox` logic and identical env var
|
||||
* names, so an endpoint deployed for the SDK demo works here unchanged.
|
||||
*
|
||||
* SERVER-ONLY. Missing/invalid URLs throw a RunpodError(500) so an unconfigured
|
||||
* op surfaces as a clear message rather than a crash.
|
||||
*/
|
||||
|
||||
import { stripDataUri, pickOutputImage } from "./base64";
|
||||
import { RunpodError, runpodCall } from "./client";
|
||||
import type {
|
||||
Img2ImgReq,
|
||||
InpaintReq,
|
||||
RunpodDetection,
|
||||
RunpodImageResult,
|
||||
TiltResult,
|
||||
TranscriptResult,
|
||||
} from "./types";
|
||||
|
||||
function envNum(v: string | undefined, fallback: number): number {
|
||||
// Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which
|
||||
// would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`.
|
||||
if (v == null || v.trim() === "") return fallback;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : fallback;
|
||||
}
|
||||
|
||||
function endpointUrl(envVar: string): string {
|
||||
const url = process.env[envVar];
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500);
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
/** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */
|
||||
async function imageOp(
|
||||
name: string,
|
||||
envVar: string,
|
||||
input: Record<string, unknown>,
|
||||
): Promise<RunpodImageResult> {
|
||||
const output = await runpodCall({ name, url: endpointUrl(envVar), input });
|
||||
return { imageBase64: pickOutputImage(output), mimeType: "image/png" };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Image endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */
|
||||
export function rpRemoveBackground(imageB64: string, model?: string): Promise<RunpodImageResult> {
|
||||
return imageOp("background-removal", "RUNPOD_BG_REMOVE_URL", {
|
||||
task: "remove-bg",
|
||||
image: imageB64,
|
||||
...(model ? { model_name: model } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the
|
||||
* op / restore pass) is authoritative — it is not overridden by any env default. */
|
||||
export function rpUpscale(
|
||||
imageB64: string,
|
||||
scale: 2 | 4,
|
||||
faceEnhance = false,
|
||||
): Promise<RunpodImageResult> {
|
||||
return imageOp("upscale", "RUNPOD_UPSCALE_URL", {
|
||||
task: "upscale",
|
||||
image: imageB64,
|
||||
scale,
|
||||
face_enhance: faceEnhance,
|
||||
});
|
||||
}
|
||||
|
||||
/** #8 DDColor B&W → colorize. */
|
||||
export function rpColorize(imageB64: string, inputSize?: number): Promise<RunpodImageResult> {
|
||||
return imageOp("colorize", "RUNPOD_COLORIZE_URL", {
|
||||
image: imageB64,
|
||||
...(inputSize ? { input_size: inputSize } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */
|
||||
export function rpInpaint(p: InpaintReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "inpaint",
|
||||
image: p.imageB64,
|
||||
mask: p.maskB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-inpaint", "RUNPOD_SD_INPAINT_URL", input);
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export function rpImg2Img(p: Img2ImgReq): Promise<RunpodImageResult> {
|
||||
const input: Record<string, unknown> = {
|
||||
task: "img2img",
|
||||
image: p.imageB64,
|
||||
prompt: p.prompt,
|
||||
strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6),
|
||||
guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7),
|
||||
num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35),
|
||||
};
|
||||
const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT;
|
||||
if (negative) input.negative_prompt = negative;
|
||||
if (p.seed != null) input.seed = p.seed;
|
||||
return imageOp("sd-img2img", "RUNPOD_SD_IMG2IMG_URL", input);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function rpDetect(
|
||||
imageB64: string,
|
||||
width: number,
|
||||
height: number,
|
||||
): Promise<RunpodDetection[]> {
|
||||
const output = await runpodCall<unknown>({
|
||||
name: "yolo-detect",
|
||||
url: endpointUrl("RUNPOD_YOLO_URL"),
|
||||
input: { image: imageB64, task: "detect" },
|
||||
});
|
||||
return normalizeDetections(output, width, height);
|
||||
}
|
||||
|
||||
function extractDetectionArray(output: unknown): unknown[] {
|
||||
if (Array.isArray(output)) return output;
|
||||
if (output && typeof output === "object") {
|
||||
const o = output as Record<string, unknown>;
|
||||
for (const key of ["detections", "predictions", "objects", "results", "boxes"]) {
|
||||
if (Array.isArray(o[key])) return o[key] as unknown[];
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */
|
||||
function firstLabel(...vals: unknown[]): string | undefined {
|
||||
for (const v of vals) {
|
||||
if (typeof v === "string" && v.trim()) return v.trim();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] {
|
||||
const out: RunpodDetection[] = [];
|
||||
for (const raw of extractDetectionArray(output)) {
|
||||
if (!raw || typeof raw !== "object") continue;
|
||||
const o = raw as Record<string, unknown>;
|
||||
// Prefer a human-readable name (ultralytics tojson puts the string in `name`
|
||||
// and a numeric index in `class`); fall back to class_<id>. Using firstLabel
|
||||
// (not `??`) also means an explicit empty-string label doesn't get kept + dropped.
|
||||
const classId = o.class_id ?? (typeof o.class === "number" ? o.class : undefined);
|
||||
const label = (
|
||||
firstLabel(o.label, o.name, o.class_name, typeof o.class === "string" ? o.class : undefined) ??
|
||||
(classId != null ? `class_${classId}` : "object")
|
||||
).toLowerCase();
|
||||
const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0;
|
||||
const box = normalizeBox(o, width, height);
|
||||
if (box && label) out.push({ label, confidence, box });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | null {
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
}
|
||||
|
||||
function asNum4(v: unknown): [number, number, number, number] | null {
|
||||
if (!Array.isArray(v) || v.length < 4) return null;
|
||||
const a = num(v[0]);
|
||||
const b = num(v[1]);
|
||||
const c = num(v[2]);
|
||||
const d = num(v[3]);
|
||||
return a === null || b === null || c === null || d === null ? null : [a, b, c, d];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a detection box to {x, y, width, height} as fractions 0..1 of the
|
||||
* image, from whatever shape the endpoint emits:
|
||||
* - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form
|
||||
* - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form
|
||||
* - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners
|
||||
* - object `{x,y,width,height}` → x/y/width/height
|
||||
* Pixel values (any component > 1) are divided by the image dims; already-
|
||||
* normalized fractions pass through.
|
||||
*/
|
||||
function normalizeBox(
|
||||
o: Record<string, unknown>,
|
||||
width: number,
|
||||
height: number,
|
||||
): RunpodDetection["box"] | null {
|
||||
const W = width || 1;
|
||||
const H = height || 1;
|
||||
const clamp01 = (n: number) => Math.max(0, Math.min(1, n));
|
||||
const frac = (x: number, y: number, w: number, h: number): RunpodDetection["box"] => {
|
||||
if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) {
|
||||
x /= W;
|
||||
y /= H;
|
||||
w /= W;
|
||||
h /= H;
|
||||
}
|
||||
return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) };
|
||||
};
|
||||
const fromXyxy = (x1: number, y1: number, x2: number, y2: number) =>
|
||||
frac(x1, y1, x2 - x1, y2 - y1);
|
||||
|
||||
// 1. Array boxes, interpreted by which key holds them.
|
||||
const xyxyArr = asNum4(o.xyxy);
|
||||
if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]);
|
||||
const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h]
|
||||
if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]);
|
||||
const boxArr = asNum4(o.box); // generic array box → assume corner form
|
||||
if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]);
|
||||
|
||||
// 2. Object boxes (either nested under `box` or directly on the detection).
|
||||
const src =
|
||||
o.box && typeof o.box === "object" && !Array.isArray(o.box)
|
||||
? (o.box as Record<string, unknown>)
|
||||
: o;
|
||||
const x1 = num(src.x1 ?? src.left);
|
||||
const y1 = num(src.y1 ?? src.top);
|
||||
const x2 = num(src.x2 ?? src.right);
|
||||
const y2 = num(src.y2 ?? src.bottom);
|
||||
if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2);
|
||||
|
||||
const x = num(src.x);
|
||||
const y = num(src.y);
|
||||
const w = num(src.width);
|
||||
const h = num(src.height);
|
||||
if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Audio / calibration endpoints
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** #2 Camera tilt (DeepSingleImageCalibration). */
|
||||
export async function rpTilt(imageB64: string): Promise<TiltResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "tilt",
|
||||
url: endpointUrl("RUNPOD_TILT_URL"),
|
||||
input: { image: imageB64 },
|
||||
});
|
||||
return {
|
||||
rollDegrees: Number(o.roll_degrees ?? 0) || 0,
|
||||
pitchDegrees: Number(o.pitch_degrees ?? 0) || 0,
|
||||
fovDegrees: Number(o.fov_degrees ?? 0) || 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */
|
||||
export async function rpTranscribe(
|
||||
audioB64: string,
|
||||
opts?: { language?: string; timestamps?: boolean; punctuation?: boolean },
|
||||
): Promise<TranscriptResult> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "transcribe",
|
||||
url: endpointUrl("RUNPOD_STT_URL"),
|
||||
input: { audio: audioB64, task: "transcribe", ...(opts ?? {}) },
|
||||
});
|
||||
const rawSegments = Array.isArray(o.segments) ? o.segments : [];
|
||||
const segments = rawSegments.map((s) => {
|
||||
const seg = (s ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
text: String(seg.text ?? ""),
|
||||
startSec: Number(seg.start_sec ?? 0) || 0,
|
||||
endSec: Number(seg.end_sec ?? 0) || 0,
|
||||
};
|
||||
});
|
||||
return {
|
||||
transcript: String(o.transcript ?? ""),
|
||||
segments: segments.length ? segments : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */
|
||||
export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> {
|
||||
const o = await runpodCall<Record<string, unknown>>({
|
||||
name: "audio-denoise",
|
||||
url: endpointUrl("RUNPOD_AUDIO_DENOISE_URL"),
|
||||
input: { audio: audioB64, task: "denoise" },
|
||||
});
|
||||
const a = o.audio ?? o.output ?? "";
|
||||
return { audioB64: stripDataUri(String(a)) };
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Request/response types for the RunPod endpoints behind the Smart Gallery.
|
||||
*
|
||||
* PROVENANCE: a faithful port of
|
||||
* `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/types.ts`.
|
||||
* Keep the two in sync — the RunPod endpoints are shared between the standalone
|
||||
* SDK demo and this CRM.
|
||||
*
|
||||
* SERVER-ONLY. Imported by src/app/api/gallery/ai/* route handlers, never by a
|
||||
* client component (which must not see a RunPod URL or key).
|
||||
*/
|
||||
|
||||
/** Normalized image result returned by every image endpoint (raw base64, no data: prefix). */
|
||||
export interface RunpodImageResult {
|
||||
imageBase64: string;
|
||||
mimeType: string;
|
||||
}
|
||||
|
||||
/** #9 SD 3.5 masked inpainting (also #11 outpaint, pre-padded). white in mask = regenerate. */
|
||||
export interface InpaintReq {
|
||||
imageB64: string;
|
||||
maskB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/** #10 SD 3.5 general prompt edit (img2img, no mask). */
|
||||
export interface Img2ImgReq {
|
||||
imageB64: string;
|
||||
prompt: string;
|
||||
negativePrompt?: string;
|
||||
strength?: number;
|
||||
guidanceScale?: number;
|
||||
steps?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* #1 YOLO construction-material classifier, normalized to the SDK's
|
||||
* `DetectedObject` shape (box as fractions 0..1 of the image).
|
||||
*/
|
||||
export interface RunpodDetection {
|
||||
label: string;
|
||||
confidence: number;
|
||||
box: { x: number; y: number; width: number; height: number };
|
||||
}
|
||||
|
||||
/** #2 DeepSingleImageCalibration — camera tilt. */
|
||||
export interface TiltResult {
|
||||
rollDegrees: number;
|
||||
pitchDegrees: number;
|
||||
fovDegrees: number;
|
||||
}
|
||||
|
||||
/** #3 Parakeet voice-to-text. */
|
||||
export interface TranscriptSegment {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
}
|
||||
export interface TranscriptResult {
|
||||
transcript: string;
|
||||
segments?: TranscriptSegment[];
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Session gate for the Smart Gallery AI routes.
|
||||
*
|
||||
* ---------------------------------------------------------------------------
|
||||
* TRUST MODEL
|
||||
* ---------------------------------------------------------------------------
|
||||
* These routes proxy a paid, rate-limited GPU backend (RunPod) using a secret
|
||||
* held only on this server. An unauthenticated route is therefore not merely an
|
||||
* information-disclosure problem — it is a billable-resource problem: anyone who
|
||||
* can reach the URL can spend the operator's GPU budget, and can use the CRM as
|
||||
* an open relay for arbitrary image/audio processing.
|
||||
*
|
||||
* The upstream SDK demo's routes (apps/web/src/app/api/ai/*) are COMPLETELY
|
||||
* unauthenticated. This module is the fix; every ported route must call it
|
||||
* before doing any work.
|
||||
*
|
||||
* WHO IS TRUSTED
|
||||
* We do not verify a JWT here and we do not hold any signing key. The single
|
||||
* source of truth for "is this caller signed in" is the Shell BFF
|
||||
* (`${BFF_ORIGIN}/api/session/context`), which owns the HttpOnly session cookie.
|
||||
* We forward the caller's raw `cookie` header to it and treat a 200 as proof of
|
||||
* a session. Consequences of that choice, stated explicitly:
|
||||
*
|
||||
* - The BFF is trusted absolutely. If it is compromised or misconfigured to
|
||||
* answer 200 for anonymous callers, these routes are open. BFF_ORIGIN must
|
||||
* therefore only ever point at an origin the operator controls.
|
||||
* - We forward the cookie header verbatim and nothing else. No Authorization
|
||||
* header, no bearer token, and never the RunPod key.
|
||||
* - NOTHING IS CACHED. A cached "yes" would keep a revoked/expired session
|
||||
* alive for the cache lifetime, so every AI request costs one BFF round
|
||||
* trip. That is deliberate: correctness over latency for a spend gate.
|
||||
* - A network failure reaching the BFF returns 503 (fail CLOSED), never 200.
|
||||
* If we cannot prove a session, we do not spend GPU budget.
|
||||
*
|
||||
* DEMO MODE
|
||||
* When the Shell is not configured (`NEXT_PUBLIC_SUPABASE_URL` unset) the CRM
|
||||
* runs on its mock portal and there is no session to check, so we allow the
|
||||
* request and log ONCE at startup-of-first-use. This is the same gate
|
||||
* `isShellConfigured()` uses for mock-vs-real auth elsewhere in the app.
|
||||
* IMPORTANT: never deploy to a public origin with the Shell unconfigured AND a
|
||||
* real RUNPOD_API_KEY present — that combination is an open, billable endpoint.
|
||||
*
|
||||
* WHAT THIS IS NOT
|
||||
* This is authentication only, not authorization. It answers "is there a valid
|
||||
* session", not "may this principal use the gallery". Per-resource policy for
|
||||
* gallery data lives in be-crm behind the `crm.gallery` resource; if these AI
|
||||
* routes ever need the same, check it there rather than re-deriving it here.
|
||||
*/
|
||||
|
||||
export type GallerySession =
|
||||
| { ok: true; principalId?: string }
|
||||
| { ok: false; status: number; error: string };
|
||||
|
||||
/** Mirrors src/lib/appshell.ts — kept local so this stays server-only. */
|
||||
function isShellConfigured(): boolean {
|
||||
return Boolean(process.env.NEXT_PUBLIC_SUPABASE_URL);
|
||||
}
|
||||
|
||||
let demoModeWarned = false;
|
||||
|
||||
/**
|
||||
* Resolve the caller's session. Returns `{ ok: true }` (optionally with the
|
||||
* principal id, used to key rate limits) or a ready-to-return failure with the
|
||||
* status and message the route should emit.
|
||||
*/
|
||||
export async function requireGallerySession(req: Request): Promise<GallerySession> {
|
||||
if (!isShellConfigured()) {
|
||||
if (!demoModeWarned) {
|
||||
demoModeWarned = true;
|
||||
console.warn(
|
||||
"[gallery-ai] Shell is not configured (NEXT_PUBLIC_SUPABASE_URL unset) — " +
|
||||
"AI routes are UNAUTHENTICATED in demo mode. Do not expose this deployment publicly " +
|
||||
"while RUNPOD_API_KEY is set.",
|
||||
);
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
const cookie = req.headers.get("cookie");
|
||||
if (!cookie) return { ok: false, status: 401, error: "Not signed in" };
|
||||
|
||||
const origin = (process.env.BFF_ORIGIN ?? "http://localhost:4000").replace(/\/$/, "");
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(`${origin}/api/session/context`, {
|
||||
headers: { cookie, accept: "application/json" },
|
||||
// Never cache an auth decision — see the trust-model note above.
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
} catch {
|
||||
// Fail closed: we could not prove a session, so we do not spend GPU budget.
|
||||
return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
}
|
||||
|
||||
if (res.status === 401) return { ok: false, status: 401, error: "Not signed in" };
|
||||
if (!res.ok) return { ok: false, status: 503, error: "Session service unavailable" };
|
||||
|
||||
// The principal id is best-effort: it only sharpens the rate-limit key, so a
|
||||
// shape we don't recognize degrades to IP-keyed limiting rather than failing.
|
||||
let principalId: string | undefined;
|
||||
try {
|
||||
const data = (await res.json()) as Record<string, unknown> | null;
|
||||
principalId = pickPrincipalId(data);
|
||||
} catch {
|
||||
/* ignore — see above */
|
||||
}
|
||||
|
||||
return principalId ? { ok: true, principalId } : { ok: true };
|
||||
}
|
||||
|
||||
function pickPrincipalId(data: Record<string, unknown> | null): string | undefined {
|
||||
if (!data) return undefined;
|
||||
const direct = data.userId ?? data.principalId ?? data.sub ?? data.id;
|
||||
if (typeof direct === "string" && direct) return direct;
|
||||
const user = data.user;
|
||||
if (user && typeof user === "object") {
|
||||
const u = user as Record<string, unknown>;
|
||||
const nested = u.id ?? u.userId ?? u.sub;
|
||||
if (typeof nested === "string" && nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rate-limit key for a request: the authenticated principal when known,
|
||||
* otherwise the first hop of `x-forwarded-for`.
|
||||
*
|
||||
* NOTE the first hop is client-controlled unless a trusted proxy overwrites the
|
||||
* header. It is good enough to throttle honest clients and casual abuse; it is
|
||||
* NOT a security boundary. The session gate above is the security boundary.
|
||||
*/
|
||||
export function rateLimitKey(req: Request, principalId?: string): string {
|
||||
if (principalId) return `u:${principalId}`;
|
||||
const xff = req.headers.get("x-forwarded-for") ?? "";
|
||||
const first = xff.split(",")[0]?.trim();
|
||||
return `ip:${first || "unknown"}`;
|
||||
}
|
||||
Reference in New Issue
Block a user