/** * Typed wrappers for each RunPod endpoint behind the Smart Gallery. * Every function reads its own `RUNPOD_*_URL` env var, sends the exact `input` * contract the model spec documents, and normalizes the response. * * PROVENANCE: a faithful port of * `advance-photo-gallery-web-sdk/apps/web/src/lib/runpod/endpoints.ts` — * identical `normalizeDetections` / `normalizeBox` logic and identical env var * names, so an endpoint deployed for the SDK demo works here unchanged. * * SERVER-ONLY. Missing/invalid URLs throw a RunpodError(500) so an unconfigured * op surfaces as a clear message rather than a crash. */ import { stripDataUri, pickOutputImage } from "./base64"; import { RunpodError, runpodCall } from "./client"; import type { Img2ImgReq, InpaintReq, RunpodDetection, RunpodImageResult, TiltResult, TranscriptResult, } from "./types"; function envNum(v: string | undefined, fallback: number): number { // Treat a blank/whitespace env var as unset — Number('') is 0 (finite), which // would otherwise send e.g. strength:0 for `RUNPOD_SD_STRENGTH=`. if (v == null || v.trim() === "") return fallback; const n = Number(v); return Number.isFinite(n) ? n : fallback; } function endpointUrl(envVar: string): string { const url = process.env[envVar]; if (!url || !/^https?:\/\//i.test(url)) { throw new RunpodError(`${envVar} is not set (or is not an http(s) URL).`, 500); } return url; } /** Run an image-in/image-out endpoint and normalize the result to base64 PNG. */ async function imageOp( name: string, envVar: string, input: Record, ): Promise { const output = await runpodCall({ name, url: endpointUrl(envVar), input }); return { imageBase64: pickOutputImage(output), mimeType: "image/png" }; } // --------------------------------------------------------------------------- // Image endpoints // --------------------------------------------------------------------------- /** #6 Background removal (U²-Net via rembg). model: u2net | u2netp | u2net_human_seg */ export function rpRemoveBackground(imageB64: string, model?: string): Promise { return imageOp("background-removal", "RUNPOD_BG_REMOVE_URL", { task: "remove-bg", image: imageB64, ...(model ? { model_name: model } : {}), }); } /** #7 Real-ESRGAN enhance/upscale (RealESRGAN_x4plus). The caller's scale (from the * op / restore pass) is authoritative — it is not overridden by any env default. */ export function rpUpscale( imageB64: string, scale: 2 | 4, faceEnhance = false, ): Promise { return imageOp("upscale", "RUNPOD_UPSCALE_URL", { task: "upscale", image: imageB64, scale, face_enhance: faceEnhance, }); } /** #8 DDColor B&W → colorize. */ export function rpColorize(imageB64: string, inputSize?: number): Promise { return imageOp("colorize", "RUNPOD_COLORIZE_URL", { image: imageB64, ...(inputSize ? { input_size: inputSize } : {}), }); } /** #9 SD 3.5 masked inpainting (sky fix / eraser / fill). Also #11 outpaint (pre-padded). */ export function rpInpaint(p: InpaintReq): Promise { const input: Record = { task: "inpaint", image: p.imageB64, mask: p.maskB64, prompt: p.prompt, strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.8), guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7), num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35), }; const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT; if (negative) input.negative_prompt = negative; if (p.seed != null) input.seed = p.seed; return imageOp("sd-inpaint", "RUNPOD_SD_INPAINT_URL", input); } /** #10 SD 3.5 general prompt edit (img2img, no mask). */ export function rpImg2Img(p: Img2ImgReq): Promise { const input: Record = { task: "img2img", image: p.imageB64, prompt: p.prompt, strength: p.strength ?? envNum(process.env.RUNPOD_SD_STRENGTH, 0.6), guidance_scale: p.guidanceScale ?? envNum(process.env.RUNPOD_SD_GUIDANCE, 7), num_inference_steps: p.steps ?? envNum(process.env.RUNPOD_SD_STEPS, 35), }; const negative = p.negativePrompt ?? process.env.RUNPOD_SD_NEGATIVE_PROMPT; if (negative) input.negative_prompt = negative; if (p.seed != null) input.seed = p.seed; return imageOp("sd-img2img", "RUNPOD_SD_IMG2IMG_URL", input); } // --------------------------------------------------------------------------- // #1 YOLO detection → SDK DetectedObject shape (box as fractions 0..1) // --------------------------------------------------------------------------- export async function rpDetect( imageB64: string, width: number, height: number, ): Promise { const output = await runpodCall({ name: "yolo-detect", url: endpointUrl("RUNPOD_YOLO_URL"), input: { image: imageB64, task: "detect" }, }); return normalizeDetections(output, width, height); } function extractDetectionArray(output: unknown): unknown[] { if (Array.isArray(output)) return output; if (output && typeof output === "object") { const o = output as Record; for (const key of ["detections", "predictions", "objects", "results", "boxes"]) { if (Array.isArray(o[key])) return o[key] as unknown[]; } } return []; } /** First non-empty STRING among the args (numbers ignored — a numeric `class` is an index, not a name). */ function firstLabel(...vals: unknown[]): string | undefined { for (const v of vals) { if (typeof v === "string" && v.trim()) return v.trim(); } return undefined; } function normalizeDetections(output: unknown, width: number, height: number): RunpodDetection[] { const out: RunpodDetection[] = []; for (const raw of extractDetectionArray(output)) { if (!raw || typeof raw !== "object") continue; const o = raw as Record; // Prefer a human-readable name (ultralytics tojson puts the string in `name` // and a numeric index in `class`); fall back to class_. Using firstLabel // (not `??`) also means an explicit empty-string label doesn't get kept + dropped. const classId = o.class_id ?? (typeof o.class === "number" ? o.class : undefined); const label = ( firstLabel(o.label, o.name, o.class_name, typeof o.class === "string" ? o.class : undefined) ?? (classId != null ? `class_${classId}` : "object") ).toLowerCase(); const confidence = Number(o.confidence ?? o.score ?? o.conf ?? 0) || 0; const box = normalizeBox(o, width, height); if (box && label) out.push({ label, confidence, box }); } return out; } function num(v: unknown): number | null { const n = Number(v); return Number.isFinite(n) ? n : null; } function asNum4(v: unknown): [number, number, number, number] | null { if (!Array.isArray(v) || v.length < 4) return null; const a = num(v[0]); const b = num(v[1]); const c = num(v[2]); const d = num(v[3]); return a === null || b === null || c === null || d === null ? null : [a, b, c, d]; } /** * Normalize a detection box to {x, y, width, height} as fractions 0..1 of the * image, from whatever shape the endpoint emits: * - `xyxy: [x1,y1,x2,y2]` (ultralytics) and generic `box: [...]` → corner form * - `xywh: [...]` and COCO `bbox: [x,y,w,h]` → x/y/width/height form * - object `{x1,y1,x2,y2}` / `{left,top,right,bottom}` (ultralytics tojson) → corners * - object `{x,y,width,height}` → x/y/width/height * Pixel values (any component > 1) are divided by the image dims; already- * normalized fractions pass through. */ function normalizeBox( o: Record, width: number, height: number, ): RunpodDetection["box"] | null { const W = width || 1; const H = height || 1; const clamp01 = (n: number) => Math.max(0, Math.min(1, n)); const frac = (x: number, y: number, w: number, h: number): RunpodDetection["box"] => { if (Math.max(Math.abs(x), Math.abs(y), Math.abs(w), Math.abs(h)) > 1) { x /= W; y /= H; w /= W; h /= H; } return { x: clamp01(x), y: clamp01(y), width: clamp01(w), height: clamp01(h) }; }; const fromXyxy = (x1: number, y1: number, x2: number, y2: number) => frac(x1, y1, x2 - x1, y2 - y1); // 1. Array boxes, interpreted by which key holds them. const xyxyArr = asNum4(o.xyxy); if (xyxyArr) return fromXyxy(xyxyArr[0], xyxyArr[1], xyxyArr[2], xyxyArr[3]); const xywhArr = asNum4(o.xywh) ?? asNum4(o.bbox); // COCO `bbox` is [x,y,w,h] if (xywhArr) return frac(xywhArr[0], xywhArr[1], xywhArr[2], xywhArr[3]); const boxArr = asNum4(o.box); // generic array box → assume corner form if (boxArr) return fromXyxy(boxArr[0], boxArr[1], boxArr[2], boxArr[3]); // 2. Object boxes (either nested under `box` or directly on the detection). const src = o.box && typeof o.box === "object" && !Array.isArray(o.box) ? (o.box as Record) : o; const x1 = num(src.x1 ?? src.left); const y1 = num(src.y1 ?? src.top); const x2 = num(src.x2 ?? src.right); const y2 = num(src.y2 ?? src.bottom); if (x1 !== null && y1 !== null && x2 !== null && y2 !== null) return fromXyxy(x1, y1, x2, y2); const x = num(src.x); const y = num(src.y); const w = num(src.width); const h = num(src.height); if (x !== null && y !== null && w !== null && h !== null) return frac(x, y, w, h); return null; } // --------------------------------------------------------------------------- // Audio / calibration endpoints // --------------------------------------------------------------------------- /** #2 Camera tilt (DeepSingleImageCalibration). */ export async function rpTilt(imageB64: string): Promise { const o = await runpodCall>({ name: "tilt", url: endpointUrl("RUNPOD_TILT_URL"), input: { image: imageB64 }, }); return { rollDegrees: Number(o.roll_degrees ?? 0) || 0, pitchDegrees: Number(o.pitch_degrees ?? 0) || 0, fovDegrees: Number(o.fov_degrees ?? 0) || 0, }; } /** #3 Voice-to-text (Parakeet). Audio must be WAV 16kHz mono PCM16. */ export async function rpTranscribe( audioB64: string, opts?: { language?: string; timestamps?: boolean; punctuation?: boolean }, ): Promise { const o = await runpodCall>({ name: "transcribe", url: endpointUrl("RUNPOD_STT_URL"), input: { audio: audioB64, task: "transcribe", ...(opts ?? {}) }, }); const rawSegments = Array.isArray(o.segments) ? o.segments : []; const segments = rawSegments.map((s) => { const seg = (s ?? {}) as Record; return { text: String(seg.text ?? ""), startSec: Number(seg.start_sec ?? 0) || 0, endSec: Number(seg.end_sec ?? 0) || 0, }; }); return { transcript: String(o.transcript ?? ""), segments: segments.length ? segments : undefined, }; } /** #12 Audio noise removal (RNNoise). Audio must be WAV 48kHz mono 16-bit PCM. */ export async function rpDenoiseAudio(audioB64: string): Promise<{ audioB64: string }> { const o = await runpodCall>({ name: "audio-denoise", url: endpointUrl("RUNPOD_AUDIO_DENOISE_URL"), input: { audio: audioB64, task: "denoise" }, }); const a = o.audio ?? o.output ?? ""; return { audioB64: stripDataUri(String(a)) }; }