Refactor code structure for improved readability and maintainability

packages/photo-sdk/src/lib/audioCapture.ts

@@ -0,0 +1,138 @@
+/**
+ * Microphone capture + WAV (PCM16) encoding — browser-only, no external deps.
+ *
+ * Powers the voice-annotation UI: record speech, optionally run it through the
+ * audio-denoise model, then transcribe. Records via MediaRecorder, then decodes
+ * and resamples with the Web Audio API to the mono sample rate each model wants
+ * (16 kHz for speech-to-text, 48 kHz for denoise) and encodes 16-bit PCM WAV.
+ *
+ * All browser globals are touched at call time (never module top-level), so
+ * importing this on the server is safe.
+ */
+
+export interface Recorder {
+  /** Stop recording, release the mic, and resolve with the recorded audio blob. */
+  stop(): Promise<Blob>;
+  /** Abort without producing a blob (still releases the mic). */
+  cancel(): void;
+}
+
+/** Begin recording from the default microphone. Rejects if mic access is denied. */
+export async function startRecording(): Promise<Recorder> {
+  if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
+    throw new Error('Microphone is not available in this browser.');
+  }
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  const recorder = new MediaRecorder(stream);
+  const chunks: BlobPart[] = [];
+  recorder.ondataavailable = (e) => {
+    if (e.data && e.data.size > 0) chunks.push(e.data);
+  };
+  const release = () => stream.getTracks().forEach((t) => t.stop());
+  recorder.start();
+
+  const collect = () => new Blob(chunks, { type: recorder.mimeType || 'audio/webm' });
+
+  return {
+    stop() {
+      return new Promise<Blob>((resolve) => {
+        recorder.onstop = () => {
+          release();
+          resolve(collect());
+        };
+        try {
+          recorder.stop();
+        } catch {
+          release();
+          resolve(collect());
+        }
+      });
+    },
+    cancel() {
+      try {
+        recorder.stop();
+      } catch {
+        /* ignore */
+      }
+      release();
+    },
+  };
+}
+
+/**
+ * Decode any recorded audio blob, downmix to mono, resample to `sampleRate`, and
+ * return base64 (no `data:` prefix) of a 16-bit PCM WAV.
+ */
+export async function blobToWavBase64(blob: Blob, sampleRate: number): Promise<string> {
+  const arrayBuf = await blob.arrayBuffer();
+  const AudioCtx =
+    typeof window !== 'undefined'
+      ? window.AudioContext ||
+        (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
+      : undefined;
+  if (!AudioCtx) throw new Error('Web Audio is not supported in this browser.');
+
+  const decodeCtx = new AudioCtx();
+  let decoded: AudioBuffer;
+  try {
+    // slice(0) — decodeAudioData detaches the buffer; keep a copy.
+    decoded = await decodeCtx.decodeAudioData(arrayBuf.slice(0));
+  } finally {
+    void decodeCtx.close();
+  }
+
+  // Downmix (multi-channel → 1) + resample via an OfflineAudioContext at the target rate.
+  const frames = Math.max(1, Math.round(decoded.duration * sampleRate));
+  const offline = new OfflineAudioContext(1, frames, sampleRate);
+  const source = offline.createBufferSource();
+  source.buffer = decoded;
+  source.connect(offline.destination);
+  source.start();
+  const rendered = await offline.startRendering();
+  return base64FromBytes(new Uint8Array(encodeWavPcm16(rendered.getChannelData(0), sampleRate)));
+}
+
+/** Turn base64 WAV (as returned by the denoise model) back into a Blob. */
+export function wavBase64ToBlob(base64: string): Blob {
+  const clean = base64.includes(',') ? base64.split(',', 2)[1]! : base64;
+  const bin = atob(clean);
+  const bytes = new Uint8Array(bin.length);
+  for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
+  return new Blob([bytes], { type: 'audio/wav' });
+}
+
+function encodeWavPcm16(samples: Float32Array, sampleRate: number): ArrayBuffer {
+  const buffer = new ArrayBuffer(44 + samples.length * 2);
+  const view = new DataView(buffer);
+  const writeStr = (offset: number, s: string) => {
+    for (let i = 0; i < s.length; i++) view.setUint8(offset + i, s.charCodeAt(i));
+  };
+  writeStr(0, 'RIFF');
+  view.setUint32(4, 36 + samples.length * 2, true);
+  writeStr(8, 'WAVE');
+  writeStr(12, 'fmt ');
+  view.setUint32(16, 16, true); // fmt chunk size
+  view.setUint16(20, 1, true); // audio format = PCM
+  view.setUint16(22, 1, true); // channels = mono
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * 2, true); // byte rate = sampleRate * blockAlign
+  view.setUint16(32, 2, true); // block align = channels * bytesPerSample
+  view.setUint16(34, 16, true); // bits per sample
+  writeStr(36, 'data');
+  view.setUint32(40, samples.length * 2, true);
+  let offset = 44;
+  for (let i = 0; i < samples.length; i++, offset += 2) {
+    const s = Math.max(-1, Math.min(1, samples[i]!));
+    view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+  return buffer;
+}
+
+function base64FromBytes(bytes: Uint8Array): string {
+  let bin = '';
+  const chunk = 0x8000;
+  for (let i = 0; i < bytes.length; i += chunk) {
+    bin += String.fromCharCode(...bytes.subarray(i, i + chunk));
+  }
+  return btoa(bin);
+}

packages/photo-sdk/src/lib/media.ts

@@ -52,6 +52,9 @@ export function createMediaItem(input: MediaInput): MediaItem {
     caption: input.caption,
     objects: input.objects,
     faces: input.faces,
+    // Preserve OCR text through the normalization path — without this it's dropped
+    // on every reload, so the analyzer re-runs OCR on the whole library each time.
+    ocrText: input.ocrText,
     analyzedAt: input.analyzedAt,
     colorPalette: input.colorPalette,
     blurScore: input.blurScore,
@@ -210,7 +213,15 @@ async function readExif(file: File): Promise<ExifMeta> {
       out.location = { lat: data.latitude, lng: data.longitude };
     }
     const exif: Record<string, string | number> = {};
-    for (const k of ['Make', 'Model', 'LensModel', 'ISO', 'FNumber', 'FocalLength'] as const) {
+    for (const k of [
+      'Make',
+      'Model',
+      'LensModel',
+      'ISO',
+      'FNumber',
+      'FocalLength',
+      'Orientation',
+    ] as const) {
       const v = data[k];
       if (typeof v === 'string' || typeof v === 'number') exif[k] = v;
     }

packages/photo-sdk/src/lib/smartAlbums.ts

@@ -1,7 +1,19 @@
 import type { Album, MediaItem, SmartRule, SmartRuleSet } from '../types';
 
+/** The user's permanent tag-rename map: canonical lowercased detector label → chosen label. */
+export type LabelAliases = Record<string, string>;
+
+/**
+ * Resolve a detected object label through the user's rename alias map. Keys are
+ * canonical lowercased detector labels ("car"); the value is the label the user
+ * renamed it to ("excavator"). Returns the original label when no alias applies.
+ */
+export function resolveLabel(label: string, aliases: LabelAliases = {}): string {
+  return aliases[label.trim().toLowerCase()] ?? label;
+}
+
 /** Evaluate a single smart rule against a media item. */
-function evalRule(item: MediaItem, rule: SmartRule): boolean {
+function evalRule(item: MediaItem, rule: SmartRule, aliases: LabelAliases = {}): boolean {
   const { field, op, value } = rule;
 
   const get = (): unknown => {
@@ -18,7 +30,9 @@ function evalRule(item: MediaItem, rule: SmartRule): boolean {
       case 'isLivePhoto': return Boolean(item.isLivePhoto);
       case 'isPanorama': return Boolean(item.isPanorama);
       case 'tag': return item.tags;
-      case 'object': return item.objectLabels;
+      // Resolve object labels through the rename map so a photo detected as the
+      // original label still matches the renamed album's rule.
+      case 'object': return item.objectLabels.map((l) => resolveLabel(l, aliases));
       case 'person': return item.personIds;
       default: return undefined;
     }
@@ -48,18 +62,26 @@ function evalRule(item: MediaItem, rule: SmartRule): boolean {
 }
 
 /** Does an item satisfy a rule set (AND/OR over its rules)? */
-export function matchesRuleSet(item: MediaItem, ruleSet: SmartRuleSet): boolean {
+export function matchesRuleSet(
+  item: MediaItem,
+  ruleSet: SmartRuleSet,
+  aliases: LabelAliases = {},
+): boolean {
   if (item.deletedAt) return false; // trashed items never appear in smart albums
   if (ruleSet.rules.length === 0) return true;
   return ruleSet.match === 'all'
-    ? ruleSet.rules.every((r) => evalRule(item, r))
-    : ruleSet.rules.some((r) => evalRule(item, r));
+    ? ruleSet.rules.every((r) => evalRule(item, r, aliases))
+    : ruleSet.rules.some((r) => evalRule(item, r, aliases));
 }
 
 /** Resolve the live member ids of a smart album from the full library. */
-export function resolveSmartAlbum(album: Album, items: MediaItem[]): string[] {
+export function resolveSmartAlbum(
+  album: Album,
+  items: MediaItem[],
+  aliases: LabelAliases = {},
+): string[] {
   if (!album.ruleSet) return [];
-  return items.filter((i) => matchesRuleSet(i, album.ruleSet!)).map((i) => i.id);
+  return items.filter((i) => matchesRuleSet(i, album.ruleSet!, aliases)).map((i) => i.id);
 }
 
 /**
@@ -141,12 +163,18 @@ function titleCase(s: string): string {
  *
  * @param minCount only surface a label once it appears on at least this many photos.
  */
-export function objectSmartAlbums(media: MediaItem[], now: number, minCount = 1): Album[] {
+export function objectSmartAlbums(
+  media: MediaItem[],
+  now: number,
+  aliases: LabelAliases = {},
+  minCount = 1,
+): Album[] {
   const counts = new Map<string, number>();
   for (const m of media) {
     if (m.deletedAt || m.hidden) continue;
-    // De-dupe labels within one item so a single photo counts once per label.
-    for (const label of new Set(m.objectLabels)) {
+    // Resolve labels through the rename map, then de-dupe within one item so a
+    // single photo counts once per (renamed) label and renamed labels regroup.
+    for (const label of new Set(m.objectLabels.map((l) => resolveLabel(l, aliases)))) {
       const key = label.trim().toLowerCase();
       if (!key) continue;
       counts.set(key, (counts.get(key) ?? 0) + 1);

packages/photo-sdk/src/lib/versions.ts

@@ -31,6 +31,7 @@ export function summarizeEdits(edits?: EditState): string[] {
     if (edits.overlays?.some((o) => o.kind === 'text')) c.push('Text');
   }
   if (edits.audio?.muted) c.push('Muted original audio');
+  if (edits.audio?.denoisedSrc) c.push('Reduced audio noise (AI)');
   if (edits.audio?.musicSrc) c.push('Added music');
   if (edits.audio?.fadeIn || edits.audio?.fadeOut) c.push('Audio fade');
   return c.length ? c : ['Edited'];

packages/photo-sdk/src/lib/videoBake.ts

@@ -173,7 +173,10 @@ export async function bakeVideo(
   try {
     const vNode = ac.createMediaElementSource(video);
     const vGain = ac.createGain();
-    vGain.gain.value = edits.audio?.muted ? 0 : (edits.audio?.originalVolume ?? 1);
+    // When an AI-denoised track was produced, mute the original and play the clean
+    // one instead (added below); otherwise use the original at its set volume.
+    const hasClean = Boolean(edits.audio?.denoisedSrc);
+    vGain.gain.value = edits.audio?.muted || hasClean ? 0 : (edits.audio?.originalVolume ?? 1);
     vNode.connect(vGain).connect(master);
   } catch {
     /* element may have no audio track */
@@ -192,6 +195,21 @@ export async function bakeVideo(
     }
   }
 
+  // AI-denoised original audio (RunPod): play it in place of the muted original.
+  let denoised: HTMLAudioElement | null = null;
+  if (edits.audio?.denoisedSrc && !edits.audio?.muted) {
+    denoised = new Audio(edits.audio.denoisedSrc);
+    denoised.crossOrigin = 'anonymous';
+    try {
+      const dNode = ac.createMediaElementSource(denoised);
+      const dGain = ac.createGain();
+      dGain.gain.value = edits.audio.originalVolume ?? 1;
+      dNode.connect(dGain).connect(master);
+    } catch {
+      /* ignore */
+    }
+  }
+
   // 6) Recorder over canvas video + mixed audio.
   const stream = canvas.captureStream(fps);
   const audioTrack = dest.stream.getAudioTracks()[0];
@@ -283,6 +301,7 @@ export async function bakeVideo(
     await seekTo(video, seg.start);
     video.playbackRate = seg.speed || 1;
     if (music && seg === segs[0]) await music.play().catch(() => {});
+    if (denoised && seg === segs[0]) await denoised.play().catch(() => {});
     await video.play().catch(() => {});
     const segOut = (seg.end - seg.start) / (seg.speed || 1);
     // Guard against a stalled decoder (autoplay blocked, boundary glitch).
@@ -310,6 +329,7 @@ export async function bakeVideo(
   }
 
   music?.pause();
+  denoised?.pause();
   if (recorder.state !== 'inactive') recorder.stop();
   await stopped;
   opts.signal?.removeEventListener('abort', onAbort);