KaushikRK99/advance-photo-gallery-web-sdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor code structure for improved readability and maintainability

Browse Source
This commit is contained in:
Kaushik Ranpriya
2026-07-22 22:58:28 +05:30
parent d1308bf149
commit bc8bf2007d
99 changed files with 4784 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/photo-sdk/src/adapters/localStorage.ts
+14
View File
@@ -43,10 +43,24 @@ export function createLocalStorageAdapter(key: string = STORAGE_KEY): StorageAda
: [],
}))
: [];
// Keep only string→string entries — never trust persisted data blindly.
const labelAliases: Record<string, string> =
parsed.labelAliases && typeof parsed.labelAliases === 'object'
? Object.fromEntries(
Object.entries(parsed.labelAliases).filter(
([k, v]) => typeof k === 'string' && typeof v === 'string',
),
)
: {};
const deletedLabels: string[] = Array.isArray(parsed.deletedLabels)
? (parsed.deletedLabels as unknown[]).filter((l): l is string => typeof l === 'string')
: [];
return {
media,
albums,
people: Array.isArray(parsed.people) ? parsed.people : [],
labelAliases,
deletedLabels,
version: typeof parsed.version === 'number' ? parsed.version : CURRENT_VERSION,
};
} catch {
packages/photo-sdk/src/adapters/types.ts
+6
View File
@@ -4,6 +4,12 @@ export interface PersistedState {
media: MediaItem[];
albums: Album[];
people: Person[];
/**
* User's permanent object-tag renames (canonical lowercased detector label →
* chosen label). Optional for backward compatibility with pre-rename data.
*/
labelAliases?: Record<string, string>;
deletedLabels?: string[];
version: number;
}
packages/photo-sdk/src/ai/types.ts
+33 -5
View File
@@ -33,18 +33,46 @@ export interface AIProvider {
image: ImageBitmap | HTMLImageElement,
op: GenerativeEditOp,
): Promise<Blob>;
/**
* Transcribe recorded speech (base64 WAV, 16 kHz mono PCM16) to text.
* Powers voice input for photo annotations / comments.
*/
transcribeAudio?(audioBase64: string): Promise<string>;
/**
* Denoise recorded audio (base64 WAV) → cleaned base64 WAV. Useful before
* transcription when the recording was made on a noisy site.
*/
denoiseAudio?(audioBase64: string): Promise<string>;
/** Estimate camera tilt (roll/pitch/fov, degrees) for auto-straightening. */
estimateTilt?(item: MediaItem, image: ImageBitmap | HTMLImageElement): Promise<TiltEstimate>;
}
/** Camera-tilt estimate (degrees). `rollDegrees` = in-plane rotation to correct. */
export interface TiltEstimate {
rollDegrees: number;
pitchDegrees: number;
fovDegrees: number;
}
export type GenerativeEditOp =
| { type: 'remove-background' }
| { type: 'replace-sky'; prompt?: string }
| { type: 'magic-eraser'; mask: ImageData }
| { type: 'generative-fill'; prompt: string; mask: ImageData }
| { type: 'replace-sky'; prompt?: string; strength?: number }
| { type: 'magic-eraser'; mask: ImageData; strength?: number }
| { type: 'generative-fill'; prompt: string; mask: ImageData; strength?: number }
| { type: 'restore' }
| { type: 'upscale'; factor: 2 | 4 }
| { type: 'colorize' }
/** Free-form natural-language edit instruction. */
| { type: 'prompt'; prompt: string };
/** Outpaint / expand-canvas: pad the image and generatively fill the new border. */
| { type: 'outpaint'; prompt?: string; factor?: number; strength?: number }
/**
* Free-form natural-language edit instruction. `strength` (0..1) controls how
* strongly the edit is applied (subtle → strong); the backend maps it to the
* appropriate model knob.
*/
| { type: 'prompt'; prompt: string; strength?: number };
/** Cosine similarity between two equal-length vectors. */
export function cosineSimilarity(a: number[], b: number[]): number {
packages/photo-sdk/src/components/AIAnalyzer.tsx
+21 -9
View File
@@ -4,12 +4,13 @@ import { useEffect, useRef } from 'react';
import type { AIProvider } from '../ai/types';
import { PET_LABELS } from '../constants';
import { resolveLabel } from '../lib/smartAlbums';
import { sanitizeOcrText } from '../lib/text';
import { useGallery, useGalleryStoreApi } from '../store/context';
import type { MediaItem } from '../types';
const CONCURRENCY = 2;
const CONFIDENCE = 0.5;
const CONFIDENCE = 0.15;
const START_DELAY_MS = 1200;
/**
@@ -33,14 +34,16 @@ export function AIAnalyzer({ provider }: { provider: AIProvider }) {
provider.detectObjects || provider.detectFaces || provider.ocr || provider.embedImage;
if (!ready || !canDetect || runningRef.current) return;
// Each capability has its OWN backfill gate (not the shared analyzedAt), so a
// capability added in a later session re-processes items analyzed before it
// existed: objects key off analyzedAt; faces off `faces === undefined`; OCR
// off `ocrText === undefined`.
// Analyze each image EXACTLY ONCE: every capability is gated on `!analyzedAt`,
// so once an item has been analyzed (and analyzedAt persisted) it is never
// re-processed on later reloads — even if an individual result field didn't
// round-trip through storage. Fresh uploads (no analyzedAt) run everything.
const needsObjects = (m: MediaItem) => !!provider.detectObjects && !m.analyzedAt;
const needsFaces = (m: MediaItem) => !!provider.detectFaces && m.faces === undefined;
const needsOcr = (m: MediaItem) => !!provider.ocr && m.ocrText === undefined;
const needsEmbedding = (m: MediaItem) => !!provider.embedImage && m.embedding === undefined;
const needsFaces = (m: MediaItem) =>
!!provider.detectFaces && !m.analyzedAt && m.faces === undefined;
const needsOcr = (m: MediaItem) => !!provider.ocr && !m.analyzedAt && m.ocrText === undefined;
const needsEmbedding = (m: MediaItem) =>
!!provider.embedImage && !m.analyzedAt && m.embedding === undefined;
// Collect items still needing analysis, NEWEST FIRST — so a freshly uploaded or
// captured photo is tagged immediately instead of waiting behind the whole library.
const collectPending = () =>
@@ -99,8 +102,17 @@ export function AIAnalyzer({ provider }: { provider: AIProvider }) {
const patch: Partial<MediaItem> = { analyzedAt: Date.now() };
if (objects) {
patch.objects = objects;
// Map each detected label through the user's rename map so future uploads
// are stored/grouped under the renamed label instead of a fresh class.
const aliases = api.getState().labelAliases;
const deleted = api.getState().deletedLabels;
patch.objectLabels = [
...new Set(objects.filter((o) => o.confidence >= CONFIDENCE).map((o) => o.label)),
...new Set(
objects
.filter((o) => o.confidence >= CONFIDENCE)
.map((o) => resolveLabel(o.label, aliases))
.filter((l) => !deleted.includes(l)),
),
];
}
if (faces) patch.faces = faces;
packages/photo-sdk/src/components/InfoPanel.tsx
+174 -1
View File
@@ -5,7 +5,9 @@ import { useEffect, useRef, useState } from 'react';
import { sourceLabel } from '../lib/classify';
import { editFilterCss, editTransformCss } from '../lib/edits';
import { formatBytes, formatDate, formatDuration, formatTime } from '../lib/format';
import { blobToWavBase64, startRecording, wavBase64ToBlob, type Recorder } from '../lib/audioCapture';
import { Icon } from '../icons';
import { useAIProvider } from './aiContext';
import { useGallery, useGalleryStoreApi } from '../store/context';
import type { MediaItem } from '../types';
@@ -49,6 +51,13 @@ export function InfoPanel() {
{formatDate(item.takenAt)} · {formatTime(item.takenAt)}
</div>
{item.kind === 'image' && !item.analyzedAt ? (
<div className="apg-info__analyzing">
<span className="apg-info__spinner" aria-hidden />
Analyzing image…
</div>
) : null}
<Row label="Kind" value={item.kind === 'video' ? 'Video' : 'Photo'} />
<Row label="Source" value={sourceLabel(item.source)} />
<Row label="Format" value={`${item.mime} (${ext})`} />
@@ -70,6 +79,10 @@ export function InfoPanel() {
.join(' · ')}
/>
) : null}
{item.exif?.LensModel ? <Row label="Lens" value={String(item.exif.LensModel)} /> : null}
{item.exif?.Orientation ? (
<Row label="Orientation" value={orientationLabel(item.exif.Orientation)} />
) : null}
{item.objectLabels.length ? (
<Chips
@@ -113,8 +126,13 @@ export function InfoPanel() {
lng={item.location.lng}
onOpen={() => api.getState().focusMap({ lat: item.location!.lat, lng: item.location!.lng })}
/>
<AddressLookup
lat={item.location.lat}
lng={item.location.lng}
onOpenMap={() => api.getState().focusMap({ lat: item.location!.lat, lng: item.location!.lng })}
/>
<div style={{ fontSize: 11, color: 'var(--apg-text-tertiary)', marginTop: 4 }}>
Click the map to open it in full.
Click the map or address to open it in full.
</div>
</>
) : null}
@@ -221,6 +239,49 @@ function Comments({ item }: { item: MediaItem }) {
return window.localStorage.getItem('apg:comment-author') || 'You';
});
const provider = useAIProvider();
const canVoice = Boolean(provider?.transcribeAudio);
const canDenoise = Boolean(provider?.denoiseAudio);
const [recording, setRecording] = useState(false);
const [denoise, setDenoise] = useState(false);
const [voiceStatus, setVoiceStatus] = useState<string | null>(null);
const recorderRef = useRef<Recorder | null>(null);
const startVoice = async () => {
setVoiceStatus(null);
try {
recorderRef.current = await startRecording();
setRecording(true);
} catch (e) {
setVoiceStatus(e instanceof Error ? e.message : 'Microphone unavailable.');
}
};
const stopVoice = async () => {
const rec = recorderRef.current;
recorderRef.current = null;
setRecording(false);
if (!rec || !provider?.transcribeAudio) return;
try {
const blob = await rec.stop();
let wav16: string;
if (denoise && provider.denoiseAudio) {
setVoiceStatus('Reducing noise…');
const wav48 = await blobToWavBase64(blob, 48000);
const cleaned = await provider.denoiseAudio(wav48);
wav16 = await blobToWavBase64(wavBase64ToBlob(cleaned), 16000);
} else {
wav16 = await blobToWavBase64(blob, 16000);
}
setVoiceStatus('Transcribing…');
const spoken = (await provider.transcribeAudio(wav16)).trim();
if (spoken) setText((prev) => (prev ? `${prev} ${spoken}` : spoken));
setVoiceStatus(null);
} catch (e) {
setVoiceStatus(e instanceof Error ? e.message : 'Could not transcribe audio.');
}
};
const post = () => {
const t = text.trim();
if (!t) return;
@@ -294,6 +355,36 @@ function Comments({ item }: { item: MediaItem }) {
rows={2}
maxLength={2000}
/>
{canVoice ? (
<div className="apg-voice">
<button
type="button"
className={`apg-btn apg-btn--small apg-voice__mic${recording ? ' apg-voice__mic--rec' : ''}`}
onClick={recording ? stopVoice : startVoice}
aria-label={recording ? 'Stop recording' : 'Record a voice comment'}
title={recording ? 'Stop & transcribe' : 'Speak your comment'}
>
<Icon name={recording ? 'check' : 'mic'} size={14} />
{recording ? 'Stop' : 'Speak'}
</button>
{canDenoise ? (
<label
className="apg-voice__denoise"
title="Clean up background noise before transcribing"
>
<input
type="checkbox"
checked={denoise}
onChange={(e) => setDenoise(e.target.checked)}
/>
Reduce noise
</label>
) : null}
<span className="apg-voice__status" aria-live="polite">
{recording ? '● Listening…' : (voiceStatus ?? '')}
</span>
</div>
) : null}
<button
type="button"
className="apg-btn apg-btn--small apg-comment-form__post"
@@ -316,6 +407,22 @@ function Row({ label, value }: { label: string; value: string }) {
);
}
// EXIF Orientation codes 1–8 → human text (construction shots are often sideways).
const ORIENTATION_LABELS: Record<number, string> = {
1: 'Normal',
2: 'Mirrored horizontal',
3: 'Rotated 180°',
4: 'Mirrored vertical',
5: 'Mirrored + 90° CCW',
6: 'Rotated 90° CW',
7: 'Mirrored + 90° CW',
8: 'Rotated 90° CCW',
};
function orientationLabel(v: string | number): string {
const n = typeof v === 'number' ? v : parseInt(String(v), 10);
return ORIENTATION_LABELS[n] ?? String(v);
}
function Chips({
label,
items,
@@ -345,6 +452,72 @@ function Chips({
);
}
/**
* "Show address" button → reverse-geocodes the GPS coords to a human-readable
* address (free OpenStreetMap Nominatim). Clicking the resolved address opens the
* full Map at that location.
*/
function AddressLookup({ lat, lng, onOpenMap }: { lat: number; lng: number; onOpenMap: () => void }) {
const [address, setAddress] = useState<string | null>(null);
const [state, setState] = useState<'idle' | 'loading' | 'error'>('idle');
const lookup = async () => {
setState('loading');
try {
const res = await fetch(
`https://nominatim.openstreetmap.org/reverse?lat=${lat}&lon=${lng}&format=json&zoom=18&addressdetails=1`,
{ headers: { Accept: 'application/json' } },
);
const data = (await res.json()) as { display_name?: string };
if (data?.display_name) {
setAddress(data.display_name);
setState('idle');
} else {
setState('error');
}
} catch {
setState('error');
}
};
if (address) {
return (
<button
type="button"
onClick={onOpenMap}
title="Open in Map"
style={{
display: 'block',
textAlign: 'left',
width: '100%',
marginTop: 6,
padding: '6px 8px',
background: 'var(--apg-bg-elevated)',
border: '1px solid var(--apg-glass-border, rgba(255,255,255,0.1))',
borderRadius: 8,
color: 'var(--apg-text)',
fontSize: 12,
cursor: 'pointer',
}}
>
📍 {address}
</button>
);
}
return (
<button
type="button"
className="apg-btn apg-btn--small"
onClick={() => void lookup()}
disabled={state === 'loading'}
style={{ marginTop: 6 }}
>
{state === 'loading' ? 'Looking up…' : state === 'error' ? 'Retry address' : 'Show address'}
</button>
);
}
/** Small non-interactive Leaflet map for the location preview (click to open full Map). */
function MiniMap({ lat, lng, onOpen }: { lat: number; lng: number; onOpen?: () => void }) {
const ref = useRef<HTMLDivElement>(null);
packages/photo-sdk/src/components/Sidebar.tsx
+35 -2
View File
@@ -7,7 +7,7 @@ import { Icon, type IconName } from '../icons';
import { useGallery, useGalleryStoreApi } from '../store/context';
import type { Album, ViewId } from '../types';
import { closeContextMenu, openContextMenu } from './ContextMenu';
import { openSecuritySettings, promptAlbumName } from './modals';
import { confirmAction, openSecuritySettings, promptAlbumName } from './modals';
interface RowProps {
icon: IconName;
@@ -161,6 +161,38 @@ export function Sidebar() {
]);
};
// Right-click an auto object album → permanently rename its tag (e.g. car → excavator).
const objectMenu = (album: Album) => (e: React.MouseEvent) => {
e.preventDefault();
const label = album.id.slice('sys:obj:'.length);
openContextMenu(e.clientX, e.clientY, [
{
label: 'Rename Tag',
icon: 'tag',
onClick: () =>
promptAlbumName(
'Rename Tag',
album.name,
(name) => api.getState().renameLabel(label, name),
{ placeholder: 'Tag name' },
),
},
{
label: 'Delete Tag',
icon: 'trash',
danger: true,
onClick: () =>
confirmAction({
title: 'Delete Tag',
message: `Remove the "${album.name}" tag? It's deleted from all photos and won't be created again.`,
confirmLabel: 'Delete',
danger: true,
onConfirm: () => api.getState().deleteLabel(label),
}),
},
]);
};
const newAlbum = () =>
promptAlbumName('New Album', '', (name) => {
const id = api.getState().createAlbum(name);
@@ -182,7 +214,7 @@ export function Sidebar() {
<Row icon="video" label="Videos" view="videos" />
<Row icon="screenshot" label="Screenshots" view="screenshots" />
<Row icon="document" label="Documents" view="sys:documents" />
<Row icon="person-circle" label="People & Pets" view="people" />
<Row icon="person-circle" label="People" view="people" />
<Row
icon="trash"
label="Recently Deleted"
@@ -251,6 +283,7 @@ export function Sidebar() {
label={a.name}
view={a.id as ViewId}
indent
onContextMenu={objectMenu(a)}
/>
))
: null}
packages/photo-sdk/src/components/editor/MaskBrush.tsx
+171
View File
@@ -0,0 +1,171 @@
'use client';
import { type PointerEvent as ReactPointerEvent, useEffect, useRef, useState } from 'react';
import { Icon } from '../../icons';
/**
* Full-screen brush overlay for masked AI edits (Magic Eraser / Generative Fill).
* The user paints over a region; on Apply we emit a binary ImageData mask where
* painted pixels are WHITE (regenerate) and everything else is BLACK (keep) —
* exactly what the inpaint backend expects (see maskToBase64 / rpInpaint).
*
* The paint canvas is sized to the image's aspect ratio (long side = BASE), then
* scaled with CSS to fit the viewport, so the returned mask lines up with the
* photo regardless of screen size.
*/
const BASE = 640;
export function MaskBrush({
src,
aspect,
title,
onCancel,
onApply,
}: {
src: string;
aspect: number; // width / height
title: string;
onCancel: () => void;
onApply: (mask: ImageData) => void;
}) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const paintingRef = useRef(false);
const lastRef = useRef<{ x: number; y: number } | null>(null);
const [brush, setBrush] = useState(48);
const [dirty, setDirty] = useState(false);
const cw = aspect >= 1 ? BASE : Math.round(BASE * aspect);
const ch = aspect >= 1 ? Math.round(BASE / aspect) : BASE;
// Escape cancels.
useEffect(() => {
const onKey = (e: KeyboardEvent) => {
if (e.key === 'Escape') onCancel();
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
}, [onCancel]);
const ctx = () => canvasRef.current?.getContext('2d') ?? null;
const toCanvas = (e: ReactPointerEvent) => {
const c = canvasRef.current;
if (!c) return { x: 0, y: 0 };
const r = c.getBoundingClientRect();
return {
x: ((e.clientX - r.left) / r.width) * c.width,
y: ((e.clientY - r.top) / r.height) * c.height,
};
};
const paintTo = (x: number, y: number) => {
const c = ctx();
if (!c) return;
c.fillStyle = 'rgba(255,60,60,0.55)';
c.strokeStyle = 'rgba(255,60,60,0.55)';
c.lineWidth = brush;
c.lineCap = 'round';
const last = lastRef.current;
if (last) {
c.beginPath();
c.moveTo(last.x, last.y);
c.lineTo(x, y);
c.stroke();
}
c.beginPath();
c.arc(x, y, brush / 2, 0, Math.PI * 2);
c.fill();
lastRef.current = { x, y };
};
const onDown = (e: ReactPointerEvent) => {
e.preventDefault();
(e.target as HTMLElement).setPointerCapture?.(e.pointerId);
paintingRef.current = true;
lastRef.current = null;
const p = toCanvas(e);
paintTo(p.x, p.y);
setDirty(true);
};
const onMove = (e: ReactPointerEvent) => {
if (!paintingRef.current) return;
const p = toCanvas(e);
paintTo(p.x, p.y);
};
const onUp = () => {
paintingRef.current = false;
lastRef.current = null;
};
const clear = () => {
const c = ctx();
if (c && canvasRef.current) c.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
setDirty(false);
};
const apply = () => {
const c = ctx();
const canvas = canvasRef.current;
if (!c || !canvas) return;
const painted = c.getImageData(0, 0, canvas.width, canvas.height);
// Binarize: any painted (alpha) pixel → opaque white, else opaque black.
const out = new ImageData(canvas.width, canvas.height);
for (let i = 0; i < painted.data.length; i += 4) {
const on = painted.data[i + 3]! > 10;
const v = on ? 255 : 0;
out.data[i] = v;
out.data[i + 1] = v;
out.data[i + 2] = v;
out.data[i + 3] = 255;
}
onApply(out);
};
return (
<div className="apg-maskbrush" role="dialog" aria-label={title}>
<div className="apg-maskbrush__title">{title}</div>
<div className="apg-maskbrush__stage" style={{ aspectRatio: `${cw} / ${ch}` }}>
<img className="apg-maskbrush__img" src={src} alt="" draggable={false} />
<canvas
ref={canvasRef}
width={cw}
height={ch}
className="apg-maskbrush__canvas"
onPointerDown={onDown}
onPointerMove={onMove}
onPointerUp={onUp}
onPointerLeave={onUp}
/>
</div>
<div className="apg-maskbrush__bar">
<label className="apg-maskbrush__brush">
Brush
<input
type="range"
min={12}
max={120}
step={2}
value={brush}
onChange={(e) => setBrush(Number(e.target.value))}
/>
</label>
<button type="button" className="apg-btn apg-btn--small" onClick={clear} disabled={!dirty}>
<Icon name="trash" size={14} /> Clear
</button>
<div style={{ flex: 1 }} />
<button type="button" className="apg-btn apg-btn--small" onClick={onCancel}>
Cancel
</button>
<button
type="button"
className="apg-btn apg-btn--primary apg-btn--small"
onClick={apply}
disabled={!dirty}
>
Apply
</button>
</div>
</div>
);
}
packages/photo-sdk/src/components/editor/PhotoEditor.tsx
+154 -3
View File
@@ -17,6 +17,8 @@ import { addToAlbumPicker, confirmAction } from '../modals';
import { useAIProvider } from '../aiContext';
import { Annotations, type AnnotationTool } from './Annotations';
import { CropBox, type CropRect } from './CropBox';
import { MaskBrush } from './MaskBrush';
import { VoiceButton } from './VoiceButton';
const RATIOS: Array<{ label: string; value: number | null }> = [
{ label: 'Free', value: null },
@@ -77,6 +79,14 @@ const AI_OPS: Array<{ label: string; op: GenerativeEditOp; icon: 'wand' | 'image
{ label: 'Replace Sky', op: { type: 'replace-sky' }, icon: 'image' },
];
/** Ops whose result varies with the "edit strength" slider (the backend maps it per model). */
const STRENGTH_OPS = new Set<GenerativeEditOp['type']>([
'prompt',
'replace-sky',
'magic-eraser',
'generative-fill',
]);
export function PhotoEditor() {
const api = useGalleryStoreApi();
const editorId = useGallery((s) => s.editorId);
@@ -91,6 +101,10 @@ export function PhotoEditor() {
const [aiError, setAiError] = useState<string | null>(null);
const [aiResultUrl, setAiResultUrl] = useState<string | null>(null);
const [aiPrompt, setAiPrompt] = useState('');
const [aiStrength, setAiStrength] = useState(0.5);
const [maskMode, setMaskMode] = useState<'magic-eraser' | 'generative-fill' | null>(null);
const [tiltBusy, setTiltBusy] = useState(false);
const [tiltError, setTiltError] = useState<string | null>(null);
const [annTool, setAnnTool] = useState<AnnotationTool>('rect');
const [annColor, setAnnColor] = useState<string>('#ff3b30');
const [cropRatio, setCropRatio] = useState<number | null>(null);
@@ -146,7 +160,11 @@ export function PhotoEditor() {
setAiError(null);
try {
const img = await loadCrossOriginImage(item.src);
const blob = await provider.generativeEdit(item, img, op);
// Attach the current "edit strength" to ops that support it.
const opToRun = STRENGTH_OPS.has(op.type)
? ({ ...op, strength: aiStrength } as GenerativeEditOp)
: op;
const blob = await provider.generativeEdit(item, img, opToRun);
aiBlobRef.current = blob;
setAiResultUrl((prev) => {
if (prev) URL.revokeObjectURL(prev);
@@ -159,6 +177,22 @@ export function PhotoEditor() {
}
};
const autoStraighten = async () => {
if (!provider?.estimateTilt) return;
setTiltBusy(true);
setTiltError(null);
try {
const img = await loadCrossOriginImage(item.src);
const t = await provider.estimateTilt(item, img);
const roll = Math.max(-45, Math.min(45, Math.round(t.rollDegrees)));
setEdits((e) => ({ ...e, straighten: roll }));
} catch (err) {
setTiltError(err instanceof Error ? err.message : 'Could not estimate tilt.');
} finally {
setTiltBusy(false);
}
};
const clearAI = () => {
aiBlobRef.current = null;
setAiError(null);
@@ -379,7 +413,7 @@ export function PhotoEditor() {
}}
>
<span className="apg-ai-spinner" style={{ width: 26, height: 26 }} />
<span style={{ fontSize: 13 }}>Generating with Gemini…</span>
<span style={{ fontSize: 13 }}>Generating…</span>
</div>
) : null}
</div>
@@ -526,6 +560,20 @@ export function PhotoEditor() {
Reset Straighten
</button>
) : null}
{provider?.estimateTilt ? (
<button
type="button"
className="apg-editor__tab"
disabled={tiltBusy}
onClick={() => void autoStraighten()}
>
<Icon name="wand" size={15} />{' '}
{tiltBusy ? 'Analyzing tilt…' : 'Auto-straighten (fix camera tilt)'}
</button>
) : null}
{tiltError ? (
<p style={{ color: '#ff6b6b', fontSize: 12, margin: 0 }}>{tiltError}</p>
) : null}
</div>
) : null}
@@ -547,6 +595,28 @@ export function PhotoEditor() {
</button>
))}
</div>
{provider?.transcribeAudio ? (
<VoiceButton
label="Speak → add text"
onText={(t) => {
setAnnotations([
...annotations,
{
id: nanoid(8),
shape: 'text',
color: annColor,
strokeWidth: 2,
x1: 0.08,
y1: 0.08,
x2: 0.55,
y2: 0.17,
text: t,
},
]);
setAnnTool('select');
}}
/>
) : null}
<div>
<div style={{ fontSize: 12, color: '#9b9ba1', marginBottom: 6 }}>Color</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
@@ -596,7 +666,7 @@ export function PhotoEditor() {
{tab === 'ai' ? (
<div style={{ display: 'flex', flexDirection: 'column', gap: 10 }}>
<p style={{ color: '#9b9ba1', fontSize: 12, margin: '0 0 4px' }}>
Generative edits powered by Gemini. Results replace the photo when you press Save.
Generative edits run through your configured AI backend. Results replace the photo when you press Save.
</p>
{AI_OPS.map((a) => (
<button
@@ -610,6 +680,35 @@ export function PhotoEditor() {
</button>
))}
<div className="apg-slider-row" style={{ marginTop: 6 }}>
<div className="apg-slider-row__head">
<span>Edit strength</span>
<span>{Math.round(aiStrength * 100)}%</span>
</div>
<input
className="apg-slider"
type="range"
min={0}
max={1}
step={0.05}
value={aiStrength}
disabled={aiBusy}
onChange={(e) => setAiStrength(Number(e.target.value))}
/>
<div
style={{
display: 'flex',
justifyContent: 'space-between',
fontSize: 11,
color: '#9b9ba1',
marginTop: 2,
}}
>
<span>Subtle</span>
<span>Strong</span>
</div>
</div>
<div style={{ marginTop: 6 }}>
<input
className="apg-modal__input"
@@ -633,6 +732,36 @@ export function PhotoEditor() {
</button>
</div>
<div style={{ height: 1, background: 'rgba(255,255,255,0.1)', margin: '4px 0' }} />
<p style={{ color: '#9b9ba1', fontSize: 12, margin: 0 }}>
Brush &amp; expand tools — inpaint / outpaint.
</p>
<button
type="button"
className="apg-editor__tab"
disabled={aiBusy}
onClick={() => setMaskMode('magic-eraser')}
>
<Icon name="wand" size={16} /> Magic Eraser (remove an object)
</button>
<button
type="button"
className="apg-editor__tab"
disabled={aiBusy}
title="Paint an area, then it fills it — type a prompt above to control what appears (optional)"
onClick={() => setMaskMode('generative-fill')}
>
<Icon name="image" size={16} /> Generative Fill (paint + prompt)
</button>
<button
type="button"
className="apg-editor__tab"
disabled={aiBusy}
onClick={() => void runAI({ type: 'outpaint', prompt: aiPrompt.trim() || undefined })}
>
<Icon name="crop" size={16} /> Expand Image (Outpaint)
</button>
{aiResultUrl ? (
<button type="button" className="apg-editor__tab" onClick={clearAI} disabled={aiBusy}>
Discard AI result
@@ -645,6 +774,28 @@ export function PhotoEditor() {
) : null}
</div>
</div>
{maskMode ? (
<MaskBrush
src={item.src}
aspect={imgAspect}
title={
maskMode === 'magic-eraser'
? 'Paint over what to remove'
: 'Paint the area to replace (uses your prompt)'
}
onCancel={() => setMaskMode(null)}
onApply={(mask) => {
const m = maskMode;
setMaskMode(null);
if (m === 'generative-fill') {
void runAI({ type: 'generative-fill', prompt: aiPrompt.trim() || 'fill naturally', mask });
} else {
void runAI({ type: 'magic-eraser', mask });
}
}}
/>
) : null}
</motion.div>
</AnimatePresence>
);
packages/photo-sdk/src/components/editor/VideoEditor.tsx
+84 -1
View File
@@ -9,6 +9,9 @@ import { useFocusTrap } from '../../hooks/useFocusTrap';
import { editFilterCss } from '../../lib/edits';
import { summarizeEdits } from '../../lib/versions';
import { bakeVideo } from '../../lib/videoBake';
import { blobToWavBase64, wavBase64ToBlob } from '../../lib/audioCapture';
import { useAIProvider } from '../aiContext';
import { VoiceButton } from './VoiceButton';
import {
normalizeSegments,
outputDuration,
@@ -91,6 +94,9 @@ export function VideoEditor() {
const [duration, setDuration] = useState(0);
const [playhead, setPlayhead] = useState(0);
const [selOverlay, setSelOverlay] = useState<string | null>(null);
const provider = useAIProvider();
const [denoiseBusy, setDenoiseBusy] = useState(false);
const [denoiseErr, setDenoiseErr] = useState<string | null>(null);
const [previewH, setPreviewH] = useState(360);
const [baking, setBaking] = useState(false);
const [progress, setProgress] = useState(0);
@@ -209,6 +215,26 @@ export function VideoEditor() {
opacity: 1,
rotation: 0,
});
const runVideoDenoise = async () => {
if (!provider?.denoiseAudio) return;
setDenoiseBusy(true);
setDenoiseErr(null);
try {
const resp = await fetch(item.src);
const blob = await resp.blob();
// Decode the video's audio track → 48 kHz mono WAV → RunPod denoise → clean WAV.
const wav48 = await blobToWavBase64(blob, 48000);
const cleaned = await provider.denoiseAudio(wav48);
const url = URL.createObjectURL(wavBase64ToBlob(cleaned));
update({ audio: { ...edits.audio, denoisedSrc: url } });
} catch (e) {
setDenoiseErr(
e instanceof Error ? e.message : 'Could not clean the audio (keep clips under ~30s).',
);
} finally {
setDenoiseBusy(false);
}
};
const addKeyframe = (id: string) => {
const o = overlays.find((x) => x.id === id);
if (!o) return;
@@ -314,6 +340,21 @@ export function VideoEditor() {
};
const filterCss = editFilterCss(edits);
// Live preview for Crop & Rotate. videoBake applies these on export; without
// mirroring them on the preview the Rotate/Flip/Crop buttons look like they do
// nothing. Quarter-turn rotations are scaled to fit the frame.
const pvRot = (((edits.rotation ?? 0) % 360) + 360) % 360;
const pvQuarter = pvRot === 90 || pvRot === 270;
const pvAspect = (item.width || 16) / (item.height || 9);
const pvFit = pvQuarter ? Math.min(pvAspect, 1 / pvAspect) : 1;
const previewTransform =
pvRot || edits.flipH || edits.flipV
? `rotate(${pvRot}deg) scale(${(edits.flipH ? -1 : 1) * pvFit}, ${(edits.flipV ? -1 : 1) * pvFit})`
: undefined;
const pvCrop = edits.crop;
const previewClip = pvCrop
? `inset(${(pvCrop.y * 100).toFixed(3)}% ${((1 - pvCrop.x - pvCrop.width) * 100).toFixed(3)}% ${((1 - pvCrop.y - pvCrop.height) * 100).toFixed(3)}% ${(pvCrop.x * 100).toFixed(3)}%)`
: undefined;
const sel = overlays.find((o) => o.id === selOverlay) ?? null;
return (
@@ -360,7 +401,15 @@ export function VideoEditor() {
controls
playsInline
crossOrigin="anonymous"
style={{ maxWidth: '100%', maxHeight: '70vh', display: 'block', filter: filterCss || undefined }}
style={{
maxWidth: '100%',
maxHeight: '70vh',
display: 'block',
filter: filterCss || undefined,
transform: previewTransform,
clipPath: previewClip,
transition: 'transform 0.15s ease',
}}
onLoadedMetadata={(e) => {
const d = e.currentTarget.duration || 0;
setDuration(d);
@@ -651,6 +700,13 @@ export function VideoEditor() {
onChange={(e) => patchOverlay(sel.id, { text: e.target.value })}
/>
</label>
<VoiceButton
label="Speak → text"
onText={(t) => {
const cur = sel.text && sel.text !== 'Your text' ? sel.text : '';
patchOverlay(sel.id, { text: cur ? `${cur} ${t}` : t });
}}
/>
<div style={{ display: 'flex', gap: 6, margin: '4px 0' }}>
{COLORS.map((c) => (
<button
@@ -794,6 +850,33 @@ export function VideoEditor() {
onChange={(e) => update({ audio: { ...edits.audio, muted: e.target.checked } })} />
Mute original audio
</label>
{provider?.denoiseAudio ? (
edits.audio?.denoisedSrc ? (
<div style={{ display: 'flex', alignItems: 'center', gap: 8, fontSize: 13 }}>
<span style={{ color: '#34c759' }}>✓ Background noise reduced (AI)</span>
<button
type="button"
className="apg-btn apg-btn--small"
onClick={() => update({ audio: { ...edits.audio, denoisedSrc: undefined } })}
>
Undo
</button>
</div>
) : (
<button
type="button"
className="apg-btn apg-btn--primary"
disabled={denoiseBusy}
onClick={() => void runVideoDenoise()}
>
<Icon name="wand" size={14} />{' '}
{denoiseBusy ? 'Cleaning audio…' : 'Reduce background noise (AI)'}
</button>
)
) : null}
{denoiseErr ? (
<p style={{ color: '#ff6b6b', fontSize: 12, margin: 0 }}>{denoiseErr}</p>
) : null}
{!edits.audio?.muted ? (
<label className="apg-vedit__row">
<span>Original volume {Math.round((edits.audio?.originalVolume ?? 1) * 100)}%</span>
packages/photo-sdk/src/components/editor/VoiceButton.tsx
+86
View File
@@ -0,0 +1,86 @@
'use client';
import { useRef, useState } from 'react';
import { blobToWavBase64, startRecording, wavBase64ToBlob, type Recorder } from '../../lib/audioCapture';
import { Icon } from '../../icons';
import { useAIProvider } from '../aiContext';
/**
* Reusable dictation button: record → (optional AI denoise) → transcribe → onText.
* Renders nothing if the AI provider can't transcribe. Used by the image-markup
* Text tool, the video Text overlay, and the Info comment box.
*/
export function VoiceButton({
onText,
denoise = false,
label = 'Speak',
size = 14,
}: {
onText: (text: string) => void;
denoise?: boolean;
label?: string;
size?: number;
}) {
const provider = useAIProvider();
const [recording, setRecording] = useState(false);
const [status, setStatus] = useState<string | null>(null);
const recRef = useRef<Recorder | null>(null);
if (!provider?.transcribeAudio) return null;
const start = async () => {
setStatus(null);
try {
recRef.current = await startRecording();
setRecording(true);
} catch (e) {
setStatus(e instanceof Error ? e.message : 'Microphone unavailable.');
}
};
const stop = async () => {
const rec = recRef.current;
recRef.current = null;
setRecording(false);
if (!rec || !provider.transcribeAudio) return;
try {
const blob = await rec.stop();
let wav: string;
if (denoise && provider.denoiseAudio) {
setStatus('Reducing noise…');
const w48 = await blobToWavBase64(blob, 48000);
const cleaned = await provider.denoiseAudio(w48);
wav = await blobToWavBase64(wavBase64ToBlob(cleaned), 16000);
} else {
wav = await blobToWavBase64(blob, 16000);
}
setStatus('Transcribing…');
const t = (await provider.transcribeAudio(wav)).trim();
if (t) onText(t);
setStatus(null);
} catch (e) {
setStatus(e instanceof Error ? e.message : 'Could not transcribe.');
}
};
return (
<span className="apg-voice">
<button
type="button"
className={`apg-btn apg-btn--small apg-voice__mic${recording ? ' apg-voice__mic--rec' : ''}`}
onClick={recording ? stop : start}
aria-label={recording ? 'Stop recording' : 'Dictate text'}
title={recording ? 'Stop & transcribe' : 'Speak to type'}
>
<Icon name={recording ? 'check' : 'mic'} size={size} />
{recording ? 'Stop' : label}
</button>
{recording || status ? (
<span className="apg-voice__status" aria-live="polite">
{recording ? '● Listening…' : status}
</span>
) : null}
</span>
);
}
packages/photo-sdk/src/components/views/CollectionsView.tsx
+39 -3
View File
@@ -4,11 +4,13 @@ import { useState } from 'react';
import { formatDay } from '../../lib/format';
import { groupByTime } from '../../lib/grouping';
import { resolveLabel } from '../../lib/smartAlbums';
import { Icon, type IconName } from '../../icons';
import { useGallery, useGalleryStoreApi } from '../../store/context';
import { albumMedia, liveMedia, objectLabelCounts } from '../../store/selectors';
import type { MediaItem, ViewId } from '../../types';
import { promptAlbumName } from '../modals';
import { openContextMenu } from '../ContextMenu';
import { confirmAction, promptAlbumName } from '../modals';
function SectionHeader({
title,
@@ -97,6 +99,7 @@ export function CollectionsView() {
const api = useGalleryStoreApi();
const media = useGallery((s) => s.media);
const albums = useGallery((s) => s.albums);
const labelAliases = useGallery((s) => s.labelAliases);
const live = liveMedia(media);
const first = (pred: (m: MediaItem) => boolean) => live.find(pred);
@@ -105,10 +108,38 @@ export function CollectionsView() {
const recentDays = groupByTime(live, 'day').slice(0, 8);
const featured = [...live].sort((a, b) => b.takenAt - a.takenAt).slice(0, 12);
const objectEntries = [...objectLabelCounts(media).entries()]
const objectEntries = [...objectLabelCounts(media, labelAliases).entries()]
.sort((a, b) => b[1] - a[1])
.slice(0, 14);
// Right-click an object card → permanently rename its tag (car → excavator).
const renameTag = (label: string) => (e: React.MouseEvent) => {
e.preventDefault();
openContextMenu(e.clientX, e.clientY, [
{
label: 'Rename Tag',
icon: 'tag',
onClick: () =>
promptAlbumName('Rename Tag', label, (name) => api.getState().renameLabel(label, name), {
placeholder: 'Tag name',
}),
},
{
label: 'Delete Tag',
icon: 'trash',
danger: true,
onClick: () =>
confirmAction({
title: 'Delete Tag',
message: `Remove the "${label}" tag? It's deleted from all photos and won't be created again.`,
confirmLabel: 'Delete',
danger: true,
onConfirm: () => api.getState().deleteLabel(label),
}),
},
]);
};
return (
<div className="apg-scroll">
<div className="apg-collections">
@@ -184,10 +215,15 @@ export function CollectionsView() {
type="button"
className="apg-pinned-card"
onClick={() => api.getState().setView(`sys:obj:${label}` as ViewId)}
onContextMenu={renameTag(label)}
aria-label={`${count} photos containing ${label}`}
>
{(() => {
const cover = live.find((m) => m.objectLabels.includes(label));
// Match on the resolved label so the cover works for items still
// stored under the original detector label.
const cover = live.find((m) =>
m.objectLabels.some((l) => resolveLabel(l, labelAliases) === label),
);
return cover ? <img src={cover.thumbnail ?? cover.src} alt="" draggable={false} /> : null;
})()}
<span className="apg-pinned-card__label" style={{ textTransform: 'capitalize' }}>
packages/photo-sdk/src/icons/index.tsx
+9 -1
View File
@@ -55,7 +55,8 @@ export type IconName =
| 'pip'
| 'tag'
| 'document'
| 'pin';
| 'pin'
| 'mic';
export interface IconProps extends SVGProps<SVGSVGElement> {
name: IconName;
@@ -83,6 +84,13 @@ export function Icon({ name, size = 20, ...rest }: IconProps) {
}
const paths: Record<IconName, JSX.Element> = {
mic: (
<>
<rect x="9" y="3" width="6" height="11" rx="3" />
<path d="M6 11a6 6 0 0 0 12 0" />
<path d="M12 17v3M9 20.5h6" />
</>
),
library: (
<>
<rect x="3" y="6" width="18" height="13" rx="2.5" />
packages/photo-sdk/src/lib/audioCapture.ts
+138
View File
@@ -0,0 +1,138 @@
/**
* Microphone capture + WAV (PCM16) encoding — browser-only, no external deps.
*
* Powers the voice-annotation UI: record speech, optionally run it through the
* audio-denoise model, then transcribe. Records via MediaRecorder, then decodes
* and resamples with the Web Audio API to the mono sample rate each model wants
* (16 kHz for speech-to-text, 48 kHz for denoise) and encodes 16-bit PCM WAV.
*
* All browser globals are touched at call time (never module top-level), so
* importing this on the server is safe.
*/
export interface Recorder {
/** Stop recording, release the mic, and resolve with the recorded audio blob. */
stop(): Promise<Blob>;
/** Abort without producing a blob (still releases the mic). */
cancel(): void;
}
/** Begin recording from the default microphone. Rejects if mic access is denied. */
export async function startRecording(): Promise<Recorder> {
if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
throw new Error('Microphone is not available in this browser.');
}
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const recorder = new MediaRecorder(stream);
const chunks: BlobPart[] = [];
recorder.ondataavailable = (e) => {
if (e.data && e.data.size > 0) chunks.push(e.data);
};
const release = () => stream.getTracks().forEach((t) => t.stop());
recorder.start();
const collect = () => new Blob(chunks, { type: recorder.mimeType || 'audio/webm' });
return {
stop() {
return new Promise<Blob>((resolve) => {
recorder.onstop = () => {
release();
resolve(collect());
};
try {
recorder.stop();
} catch {
release();
resolve(collect());
}
});
},
cancel() {
try {
recorder.stop();
} catch {
/* ignore */
}
release();
},
};
}
/**
* Decode any recorded audio blob, downmix to mono, resample to `sampleRate`, and
* return base64 (no `data:` prefix) of a 16-bit PCM WAV.
*/
export async function blobToWavBase64(blob: Blob, sampleRate: number): Promise<string> {
const arrayBuf = await blob.arrayBuffer();
const AudioCtx =
typeof window !== 'undefined'
? window.AudioContext ||
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
: undefined;
if (!AudioCtx) throw new Error('Web Audio is not supported in this browser.');
const decodeCtx = new AudioCtx();
let decoded: AudioBuffer;
try {
// slice(0) — decodeAudioData detaches the buffer; keep a copy.
decoded = await decodeCtx.decodeAudioData(arrayBuf.slice(0));
} finally {
void decodeCtx.close();
}
// Downmix (multi-channel → 1) + resample via an OfflineAudioContext at the target rate.
const frames = Math.max(1, Math.round(decoded.duration * sampleRate));
const offline = new OfflineAudioContext(1, frames, sampleRate);
const source = offline.createBufferSource();
source.buffer = decoded;
source.connect(offline.destination);
source.start();
const rendered = await offline.startRendering();
return base64FromBytes(new Uint8Array(encodeWavPcm16(rendered.getChannelData(0), sampleRate)));
}
/** Turn base64 WAV (as returned by the denoise model) back into a Blob. */
export function wavBase64ToBlob(base64: string): Blob {
const clean = base64.includes(',') ? base64.split(',', 2)[1]! : base64;
const bin = atob(clean);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
return new Blob([bytes], { type: 'audio/wav' });
}
function encodeWavPcm16(samples: Float32Array, sampleRate: number): ArrayBuffer {
const buffer = new ArrayBuffer(44 + samples.length * 2);
const view = new DataView(buffer);
const writeStr = (offset: number, s: string) => {
for (let i = 0; i < s.length; i++) view.setUint8(offset + i, s.charCodeAt(i));
};
writeStr(0, 'RIFF');
view.setUint32(4, 36 + samples.length * 2, true);
writeStr(8, 'WAVE');
writeStr(12, 'fmt ');
view.setUint32(16, 16, true); // fmt chunk size
view.setUint16(20, 1, true); // audio format = PCM
view.setUint16(22, 1, true); // channels = mono
view.setUint32(24, sampleRate, true);
view.setUint32(28, sampleRate * 2, true); // byte rate = sampleRate * blockAlign
view.setUint16(32, 2, true); // block align = channels * bytesPerSample
view.setUint16(34, 16, true); // bits per sample
writeStr(36, 'data');
view.setUint32(40, samples.length * 2, true);
let offset = 44;
for (let i = 0; i < samples.length; i++, offset += 2) {
const s = Math.max(-1, Math.min(1, samples[i]!));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
}
return buffer;
}
function base64FromBytes(bytes: Uint8Array): string {
let bin = '';
const chunk = 0x8000;
for (let i = 0; i < bytes.length; i += chunk) {
bin += String.fromCharCode(...bytes.subarray(i, i + chunk));
}
return btoa(bin);
}
packages/photo-sdk/src/lib/media.ts
+12 -1
View File
@@ -52,6 +52,9 @@ export function createMediaItem(input: MediaInput): MediaItem {
caption: input.caption,
objects: input.objects,
faces: input.faces,
// Preserve OCR text through the normalization path — without this it's dropped
// on every reload, so the analyzer re-runs OCR on the whole library each time.
ocrText: input.ocrText,
analyzedAt: input.analyzedAt,
colorPalette: input.colorPalette,
blurScore: input.blurScore,
@@ -210,7 +213,15 @@ async function readExif(file: File): Promise<ExifMeta> {
out.location = { lat: data.latitude, lng: data.longitude };
}
const exif: Record<string, string | number> = {};
for (const k of ['Make', 'Model', 'LensModel', 'ISO', 'FNumber', 'FocalLength'] as const) {
for (const k of [
'Make',
'Model',
'LensModel',
'ISO',
'FNumber',
'FocalLength',
'Orientation',
] as const) {
const v = data[k];
if (typeof v === 'string' || typeof v === 'number') exif[k] = v;
}
packages/photo-sdk/src/lib/smartAlbums.ts
+38 -10
View File
@@ -1,7 +1,19 @@
import type { Album, MediaItem, SmartRule, SmartRuleSet } from '../types';
/** The user's permanent tag-rename map: canonical lowercased detector label → chosen label. */
export type LabelAliases = Record<string, string>;
/**
* Resolve a detected object label through the user's rename alias map. Keys are
* canonical lowercased detector labels ("car"); the value is the label the user
* renamed it to ("excavator"). Returns the original label when no alias applies.
*/
export function resolveLabel(label: string, aliases: LabelAliases = {}): string {
return aliases[label.trim().toLowerCase()] ?? label;
}
/** Evaluate a single smart rule against a media item. */
function evalRule(item: MediaItem, rule: SmartRule): boolean {
function evalRule(item: MediaItem, rule: SmartRule, aliases: LabelAliases = {}): boolean {
const { field, op, value } = rule;
const get = (): unknown => {
@@ -18,7 +30,9 @@ function evalRule(item: MediaItem, rule: SmartRule): boolean {
case 'isLivePhoto': return Boolean(item.isLivePhoto);
case 'isPanorama': return Boolean(item.isPanorama);
case 'tag': return item.tags;
case 'object': return item.objectLabels;
// Resolve object labels through the rename map so a photo detected as the
// original label still matches the renamed album's rule.
case 'object': return item.objectLabels.map((l) => resolveLabel(l, aliases));
case 'person': return item.personIds;
default: return undefined;
}
@@ -48,18 +62,26 @@ function evalRule(item: MediaItem, rule: SmartRule): boolean {
}
/** Does an item satisfy a rule set (AND/OR over its rules)? */
export function matchesRuleSet(item: MediaItem, ruleSet: SmartRuleSet): boolean {
export function matchesRuleSet(
item: MediaItem,
ruleSet: SmartRuleSet,
aliases: LabelAliases = {},
): boolean {
if (item.deletedAt) return false; // trashed items never appear in smart albums
if (ruleSet.rules.length === 0) return true;
return ruleSet.match === 'all'
? ruleSet.rules.every((r) => evalRule(item, r))
: ruleSet.rules.some((r) => evalRule(item, r));
? ruleSet.rules.every((r) => evalRule(item, r, aliases))
: ruleSet.rules.some((r) => evalRule(item, r, aliases));
}
/** Resolve the live member ids of a smart album from the full library. */
export function resolveSmartAlbum(album: Album, items: MediaItem[]): string[] {
export function resolveSmartAlbum(
album: Album,
items: MediaItem[],
aliases: LabelAliases = {},
): string[] {
if (!album.ruleSet) return [];
return items.filter((i) => matchesRuleSet(i, album.ruleSet!)).map((i) => i.id);
return items.filter((i) => matchesRuleSet(i, album.ruleSet!, aliases)).map((i) => i.id);
}
/**
@@ -141,12 +163,18 @@ function titleCase(s: string): string {
*
* @param minCount only surface a label once it appears on at least this many photos.
*/
export function objectSmartAlbums(media: MediaItem[], now: number, minCount = 1): Album[] {
export function objectSmartAlbums(
media: MediaItem[],
now: number,
aliases: LabelAliases = {},
minCount = 1,
): Album[] {
const counts = new Map<string, number>();
for (const m of media) {
if (m.deletedAt || m.hidden) continue;
// De-dupe labels within one item so a single photo counts once per label.
for (const label of new Set(m.objectLabels)) {
// Resolve labels through the rename map, then de-dupe within one item so a
// single photo counts once per (renamed) label and renamed labels regroup.
for (const label of new Set(m.objectLabels.map((l) => resolveLabel(l, aliases)))) {
const key = label.trim().toLowerCase();
if (!key) continue;
counts.set(key, (counts.get(key) ?? 0) + 1);
packages/photo-sdk/src/lib/versions.ts
+1
View File
@@ -31,6 +31,7 @@ export function summarizeEdits(edits?: EditState): string[] {
if (edits.overlays?.some((o) => o.kind === 'text')) c.push('Text');
}
if (edits.audio?.muted) c.push('Muted original audio');
if (edits.audio?.denoisedSrc) c.push('Reduced audio noise (AI)');
if (edits.audio?.musicSrc) c.push('Added music');
if (edits.audio?.fadeIn || edits.audio?.fadeOut) c.push('Audio fade');
return c.length ? c : ['Edited'];
packages/photo-sdk/src/lib/videoBake.ts
+21 -1
View File
@@ -173,7 +173,10 @@ export async function bakeVideo(
try {
const vNode = ac.createMediaElementSource(video);
const vGain = ac.createGain();
vGain.gain.value = edits.audio?.muted ? 0 : (edits.audio?.originalVolume ?? 1);
// When an AI-denoised track was produced, mute the original and play the clean
// one instead (added below); otherwise use the original at its set volume.
const hasClean = Boolean(edits.audio?.denoisedSrc);
vGain.gain.value = edits.audio?.muted || hasClean ? 0 : (edits.audio?.originalVolume ?? 1);
vNode.connect(vGain).connect(master);
} catch {
/* element may have no audio track */
@@ -192,6 +195,21 @@ export async function bakeVideo(
}
}
// AI-denoised original audio (RunPod): play it in place of the muted original.
let denoised: HTMLAudioElement | null = null;
if (edits.audio?.denoisedSrc && !edits.audio?.muted) {
denoised = new Audio(edits.audio.denoisedSrc);
denoised.crossOrigin = 'anonymous';
try {
const dNode = ac.createMediaElementSource(denoised);
const dGain = ac.createGain();
dGain.gain.value = edits.audio.originalVolume ?? 1;
dNode.connect(dGain).connect(master);
} catch {
/* ignore */
}
}
// 6) Recorder over canvas video + mixed audio.
const stream = canvas.captureStream(fps);
const audioTrack = dest.stream.getAudioTracks()[0];
@@ -283,6 +301,7 @@ export async function bakeVideo(
await seekTo(video, seg.start);
video.playbackRate = seg.speed || 1;
if (music && seg === segs[0]) await music.play().catch(() => {});
if (denoised && seg === segs[0]) await denoised.play().catch(() => {});
await video.play().catch(() => {});
const segOut = (seg.end - seg.start) / (seg.speed || 1);
// Guard against a stalled decoder (autoplay blocked, boundary glitch).
@@ -310,6 +329,7 @@ export async function bakeVideo(
}
music?.pause();
denoised?.pause();
if (recorder.state !== 'inactive') recorder.stop();
await stopped;
opts.signal?.removeEventListener('abort', onAbort);
packages/photo-sdk/src/store/selectors.ts
+16 -9
View File
@@ -1,4 +1,5 @@
import { matchesRuleSet, resolveSmartAlbum } from '../lib/smartAlbums';
import { matchesRuleSet, resolveLabel, resolveSmartAlbum } from '../lib/smartAlbums';
import type { LabelAliases } from '../lib/smartAlbums';
import type { Album, MediaItem } from '../types';
import type { GalleryState, GridFilter } from './store';
@@ -18,9 +19,9 @@ export function albumById(albums: Album[], id: string): Album | undefined {
}
/** Resolve the members of any album (smart albums are computed live). */
export function albumMedia(album: Album, media: MediaItem[]): MediaItem[] {
export function albumMedia(album: Album, media: MediaItem[], aliases: LabelAliases = {}): MediaItem[] {
if (album.kind === 'smart' && album.ruleSet) {
const ids = new Set(resolveSmartAlbum(album, media));
const ids = new Set(resolveSmartAlbum(album, media, aliases));
return liveMedia(media)
.filter((m) => ids.has(m.id))
.sort((a, b) => b.takenAt - a.takenAt);
@@ -73,7 +74,7 @@ export function searchMedia(items: MediaItem[], query: string): MediaItem[] {
* grid filter, search and object-focus. Returns reverse-chronological order.
*/
export function mediaForView(state: GalleryState): MediaItem[] {
const { media, albums, view, gridFilter, searchQuery, objectFocus, tagFocus, personFocus, searchPreset, semanticResults, recentlyViewed } =
const { media, albums, view, gridFilter, searchQuery, objectFocus, tagFocus, personFocus, searchPreset, semanticResults, recentlyViewed, labelAliases } =
state;
let items: MediaItem[];
// When a search ranks results (keyword + semantic), preserve that order
@@ -103,7 +104,7 @@ export function mediaForView(state: GalleryState): MediaItem[] {
items = liveMedia(media);
} else if (view.startsWith('album:') || view.startsWith('sys:')) {
const album = albumById(albums, view);
items = album ? albumMedia(album, media) : [];
items = album ? albumMedia(album, media, labelAliases) : [];
} else if (view === 'favourites') {
items = liveMedia(media).filter((m) => m.favorite);
} else if (view === 'recently-saved') {
@@ -120,7 +121,10 @@ export function mediaForView(state: GalleryState): MediaItem[] {
items = applyGridFilter(items, gridFilter);
}
if (objectFocus) {
items = items.filter((m) => m.objectLabels.includes(objectFocus));
// Match on the resolved (renamed) label so focusing a renamed object still
// surfaces photos whose stored label is the original detector label.
const focus = resolveLabel(objectFocus, labelAliases);
items = items.filter((m) => m.objectLabels.some((l) => resolveLabel(l, labelAliases) === focus));
}
if (tagFocus) {
items = items.filter((m) => m.tags.includes(tagFocus));
@@ -153,11 +157,14 @@ export function mediaForView(state: GalleryState): MediaItem[] {
return items;
}
/** Distinct object labels across the live library (for AI auto-albums). */
export function objectLabelCounts(media: MediaItem[]): Map<string, number> {
/** Distinct object labels across the live library (for AI auto-albums), resolved
* through the user's rename map so renamed tags collapse into one bucket. */
export function objectLabelCounts(media: MediaItem[], aliases: LabelAliases = {}): Map<string, number> {
const counts = new Map<string, number>();
for (const m of liveMedia(media)) {
for (const label of m.objectLabels) {
// De-dupe per item after resolving so two labels renamed to the same value
// only count that photo once.
for (const label of new Set(m.objectLabels.map((l) => resolveLabel(l, aliases)))) {
counts.set(label, (counts.get(label) ?? 0) + 1);
}
}
packages/photo-sdk/src/store/store.ts
+86 -3
View File
@@ -97,6 +97,14 @@ export interface GalleryState {
media: MediaItem[];
albums: Album[];
people: Person[];
/**
* User's permanent object-tag renames. Key = canonical lowercased detector
* label (e.g. "car"), value = the label the user renamed it to ("excavator").
* Persisted, applied to future uploads, and resolved wherever labels are grouped.
*/
labelAliases: Record<string, string>;
/** Object/material tags the user deleted — stripped from photos + hidden from future detection. */
deletedLabels: string[];
// Navigation / view
view: ViewId;
@@ -200,6 +208,14 @@ export interface GalleryState {
syncObjectAlbums: () => void;
/** Rename a person/pet group (empty name clears it back to unnamed). */
renamePerson: (id: PersonId, name: string) => void;
/**
* Permanently rename a detected object tag (e.g. "car" → "excavator"). Records a
* persisted alias, backfills existing items' objectLabels, and rebuilds object
* albums so the rename applies to past photos and every future upload.
*/
renameLabel: (from: string, to: string) => void;
/** Permanently delete an object/material tag (removes it from photos + future detections). */
deleteLabel: (label: string) => void;
// Recently Deleted lock
setLockPassword: (password: string) => Promise<void>;
@@ -287,10 +303,10 @@ export function createGalleryStore(options: CreateStoreOptions) {
if (!adapter) return;
if (persistTimer) clearTimeout(persistTimer);
persistTimer = setTimeout(() => {
const { media, albums, people } = get();
const { media, albums, people, labelAliases, deletedLabels } = get();
// Only persist user-owned albums; smart/system albums are regenerated.
const userAlbums = albums.filter((a) => !a.system);
void adapter!.save({ media, albums: userAlbums, people, version: 1 });
void adapter!.save({ media, albums: userAlbums, people, labelAliases, deletedLabels, version: 1 });
}, 400);
};
@@ -302,6 +318,8 @@ export function createGalleryStore(options: CreateStoreOptions) {
media: options.initialMedia ?? [],
albums: [...defaultSystemAlbums(now), ...(options.initialAlbums ?? [])],
people: options.initialPeople ?? [],
labelAliases: {},
deletedLabels: [],
view: 'library',
libraryScale: 'all',
@@ -349,6 +367,8 @@ export function createGalleryStore(options: CreateStoreOptions) {
media: seedEmptyBackend ? seed : loaded.media,
albums: [...defaultSystemAlbums(ts), ...loaded.albums.filter((al) => !al.system)],
people: loaded.people ?? [],
labelAliases: loaded.labelAliases ?? {},
deletedLabels: loaded.deletedLabels ?? [],
ready: true,
aiAvailable: Boolean(ai),
});
@@ -427,6 +447,9 @@ export function createGalleryStore(options: CreateStoreOptions) {
// tags objects/faces/OCR/embeddings in the background.
get().addMedia(items);
if (albumId) get().addToAlbum(albumId, items.map((i) => i.id));
// Surface the just-uploaded photo with its Info panel open, so the user
// watches the analysis run + sees its results without hunting for it.
set({ infoOpen: true, selection: new Set([items[0]!.id]) });
}
return items.map((i) => i.id);
},
@@ -880,7 +903,7 @@ export function createGalleryStore(options: CreateStoreOptions) {
// Replace ONLY the sys:obj:* set; keep default system albums + user albums.
// Membership stays live via resolveSmartAlbum, so no per-item bookkeeping and
// no persist() needed (system albums are stripped on save + regenerated on load).
const generated = objectSmartAlbums(get().media, Date.now());
const generated = objectSmartAlbums(get().media, Date.now(), get().labelAliases);
set((s) => {
const others = s.albums.filter((a) => !a.id.startsWith('sys:obj:'));
// Skip the state write if the label SET is unchanged (avoids render churn).
@@ -899,6 +922,66 @@ export function createGalleryStore(options: CreateStoreOptions) {
}));
persist();
},
renameLabel(from, to) {
const key = from.trim().toLowerCase();
const value = to.trim().toLowerCase();
if (!key) return;
set((s) => {
const labelAliases: Record<string, string> = { ...s.labelAliases };
// Empty / unchanged name clears the alias (renames the tag back to itself).
const clearing = !value || value === key;
if (clearing) {
delete labelAliases[key];
} else {
labelAliases[key] = value;
// Re-point any earlier aliases that resolved to `key` so renaming an
// already-renamed tag keeps mapping the original detector label(s).
for (const k of Object.keys(labelAliases)) {
if (k !== key && labelAliases[k] === key) labelAliases[k] = value;
}
}
// Backfill existing items so past photos regroup under the new label and
// stay searchable by it (search reads objectLabels directly).
const target = clearing ? key : value;
const media = s.media.map((m) => {
if (!m.objectLabels.some((l) => l.trim().toLowerCase() === key)) return m;
const mapped = [
...new Set(
m.objectLabels.map((l) => (l.trim().toLowerCase() === key ? target : l)),
),
];
return { ...m, objectLabels: mapped };
});
return { labelAliases, media };
});
// Rebuild the sys:obj:* albums so the sidebar/Objects re-title live.
get().syncObjectAlbums();
persist();
},
deleteLabel(label) {
const key = label.trim().toLowerCase();
if (!key) return;
set((s) => {
const deletedLabels = s.deletedLabels.includes(key)
? s.deletedLabels
: [...s.deletedLabels, key];
// Drop any aliases pointing at this label so it can't reappear via a rename.
const labelAliases: Record<string, string> = { ...s.labelAliases };
for (const k of Object.keys(labelAliases)) {
if (k === key || labelAliases[k] === key) delete labelAliases[k];
}
// Strip the label from every photo so its album empties and it stops matching.
const media = s.media.map((m) =>
m.objectLabels.some((l) => l.trim().toLowerCase() === key)
? { ...m, objectLabels: m.objectLabels.filter((l) => l.trim().toLowerCase() !== key) }
: m,
);
return { deletedLabels, labelAliases, media };
});
get().syncObjectAlbums();
persist();
},
async setLockPassword(password) {
const hash = await hashPassword(password);
packages/photo-sdk/src/styles/sdk.css
+49
View File
@@ -1659,6 +1659,55 @@
.apg-comment-form__author:focus,
.apg-comment-form__input:focus { outline: none; border-color: var(--apg-accent); }
.apg-comment-form__post { align-self: flex-end; }
.apg-voice { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; }
.apg-voice__mic { display: inline-flex; align-items: center; gap: 5px; }
.apg-voice__mic--rec {
background: var(--apg-danger, #e5484d);
border-color: var(--apg-danger, #e5484d);
color: #fff;
}
.apg-voice__denoise {
display: inline-flex; align-items: center; gap: 5px;
font-size: 12px; opacity: 0.85; cursor: pointer; user-select: none;
}
.apg-voice__denoise input { accent-color: var(--apg-accent); }
.apg-voice__status { font-size: 12px; opacity: 0.7; min-height: 1em; }
.apg-info__analyzing {
display: flex; align-items: center; gap: 8px;
font-size: 13px; font-weight: 500; color: var(--apg-accent);
padding: 8px 10px; margin: 2px 0 4px;
background: color-mix(in srgb, var(--apg-accent) 10%, transparent);
border-radius: 8px;
}
.apg-info__spinner {
width: 14px; height: 14px; flex: none;
border: 2px solid currentColor; border-top-color: transparent;
border-radius: 50%; animation: apg-spin 0.8s linear infinite;
}
@keyframes apg-spin { to { transform: rotate(360deg); } }
@media (prefers-reduced-motion: reduce) { .apg-info__spinner { animation: none; } }
.apg-maskbrush {
position: fixed; inset: 0; z-index: 200;
background: rgba(0, 0, 0, 0.85);
display: flex; flex-direction: column; align-items: center; justify-content: center;
gap: 12px; padding: 20px;
}
.apg-maskbrush__title { color: #fff; font-size: 14px; font-weight: 600; }
.apg-maskbrush__stage {
position: relative; width: 100%;
max-width: min(90vw, 900px); max-height: 68vh;
}
.apg-maskbrush__img,
.apg-maskbrush__canvas {
position: absolute; inset: 0; width: 100%; height: 100%; border-radius: 8px;
}
.apg-maskbrush__img { object-fit: contain; user-select: none; -webkit-user-drag: none; }
.apg-maskbrush__canvas { cursor: crosshair; touch-action: none; }
.apg-maskbrush__bar {
display: flex; align-items: center; gap: 10px; flex-wrap: wrap;
width: 100%; max-width: min(90vw, 900px);
}
.apg-maskbrush__brush { display: inline-flex; align-items: center; gap: 6px; color: #fff; font-size: 12px; }
/* ---- Versions & Audit browser view ---- */
.apg-audit-list { display: flex; flex-direction: column; gap: 8px; }
packages/photo-sdk/src/types.ts
+2
View File
@@ -91,6 +91,8 @@ export interface EditState {
/** Master fade in / out over the whole exported clip, in seconds. */
fadeIn?: number;
fadeOut?: number;
/** AI-denoised audio (RunPod); replaces the original track on export when set. */
denoisedSrc?: string;
};
}