407 lines
13 KiB
JavaScript
407 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
// Sync community Codex pets from the public catalogs into the local
|
|
// `${CODEX_HOME:-$HOME/.codex}/pets/` registry that the daemon scans
|
|
// in `apps/daemon/src/codex-pets.ts`. Once synced, every pet shows up
|
|
// under Settings → Pets → Recently hatched and can be adopted with a
|
|
// single click — no manual `pet.json` / `spritesheet.webp` upload.
|
|
//
|
|
// Sources:
|
|
// - Codex Pet Share (https://codex-pet-share.pages.dev) — paginated
|
|
// Supabase Functions endpoint, ~170 pets at the time of writing.
|
|
// - j20 Hatchery (https://j20.nz/hatchery) — single-shot
|
|
// JSON catalog, ~30 pets at the time of writing.
|
|
//
|
|
// Both catalogs serve a `pet.json` (Codex pet contract) and a
|
|
// `spritesheet.webp` (8x9 atlas) per pet, so we just persist them to
|
|
// disk in the canonical Codex layout.
|
|
//
|
|
// Usage:
|
|
// node --experimental-strip-types scripts/sync-community-pets.ts
|
|
// node --experimental-strip-types scripts/sync-community-pets.ts --out /tmp/pets
|
|
// node --experimental-strip-types scripts/sync-community-pets.ts --source petshare
|
|
// node --experimental-strip-types scripts/sync-community-pets.ts --force
|
|
//
|
|
// Flags:
|
|
// --out <dir> Destination root. Defaults to
|
|
// `${CODEX_HOME:-$HOME/.codex}/pets`.
|
|
// --source <name> 'petshare' | 'hatchery' | 'all' (default).
|
|
// --force Re-download pets that already have a folder.
|
|
// --limit <n> Stop after N pets per source (handy for smoke
|
|
// tests).
|
|
// --concurrency <n> Parallel downloads. Defaults to 6.
|
|
// --no-pet-share Skip the petshare catalog.
|
|
// --no-hatchery Skip the hatchery catalog.
|
|
|
|
import { mkdir, stat, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import os from 'node:os';
|
|
|
|
const PETSHARE_BASE = 'https://ihzwckyzfcuktrljwpha.supabase.co/functions/v1/petshare';
|
|
const HATCHERY_LIST = 'https://j20.nz/hatchery/api/pets.json';
|
|
|
|
interface Args {
|
|
out: string;
|
|
sources: Set<'petshare' | 'hatchery'>;
|
|
force: boolean;
|
|
limit: number | null;
|
|
concurrency: number;
|
|
}
|
|
|
|
interface PetTask {
|
|
source: 'petshare' | 'hatchery';
|
|
// Slug-safe folder name under <out>/.
|
|
folder: string;
|
|
// Manifest written verbatim to <folder>/pet.json.
|
|
manifest: Record<string, unknown>;
|
|
// URL of the spritesheet binary.
|
|
spritesheetUrl: string;
|
|
// Detected file extension ('webp' | 'png' | 'gif').
|
|
spritesheetExt: string;
|
|
}
|
|
|
|
function parseArgs(argv: string[]): Args {
|
|
const home = process.env.CODEX_HOME?.trim() || path.join(os.homedir(), '.codex');
|
|
const args: Args = {
|
|
out: path.join(home, 'pets'),
|
|
sources: new Set(['petshare', 'hatchery']),
|
|
force: false,
|
|
limit: null,
|
|
concurrency: 6,
|
|
};
|
|
for (let i = 2; i < argv.length; i++) {
|
|
const flag = argv[i];
|
|
const next = (): string => {
|
|
const v = argv[++i];
|
|
if (!v) throw new Error(`flag ${flag} expects a value`);
|
|
return v;
|
|
};
|
|
switch (flag) {
|
|
case '--out':
|
|
args.out = path.resolve(next());
|
|
break;
|
|
case '--source': {
|
|
const value = next();
|
|
if (value === 'all') {
|
|
args.sources = new Set(['petshare', 'hatchery']);
|
|
} else if (value === 'petshare' || value === 'hatchery') {
|
|
args.sources = new Set([value]);
|
|
} else {
|
|
throw new Error(`unknown --source value: ${value}`);
|
|
}
|
|
break;
|
|
}
|
|
case '--no-pet-share':
|
|
args.sources.delete('petshare');
|
|
break;
|
|
case '--no-hatchery':
|
|
args.sources.delete('hatchery');
|
|
break;
|
|
case '--force':
|
|
args.force = true;
|
|
break;
|
|
case '--limit':
|
|
args.limit = Math.max(1, Number.parseInt(next(), 10));
|
|
break;
|
|
case '--concurrency':
|
|
args.concurrency = Math.max(1, Number.parseInt(next(), 10));
|
|
break;
|
|
case '-h':
|
|
case '--help':
|
|
printHelp();
|
|
process.exit(0);
|
|
default:
|
|
throw new Error(`unknown flag: ${flag}`);
|
|
}
|
|
}
|
|
return args;
|
|
}
|
|
|
|
function printHelp(): void {
|
|
console.log(`Sync community Codex pets into ~/.codex/pets
|
|
|
|
Usage:
|
|
node --experimental-strip-types scripts/sync-community-pets.ts [flags]
|
|
|
|
Flags:
|
|
--out <dir> Destination root (default: $CODEX_HOME/pets or ~/.codex/pets)
|
|
--source <name> petshare | hatchery | all (default: all)
|
|
--no-pet-share Skip the Codex Pet Share catalog
|
|
--no-hatchery Skip the j20 Hatchery catalog
|
|
--force Re-download pets that already exist on disk
|
|
--limit <n> Cap each source at N pets (for smoke tests)
|
|
--concurrency <n> Parallel downloads (default: 6)
|
|
-h, --help Show this message`);
|
|
}
|
|
|
|
function sanitizeFolder(value: string): string {
|
|
return String(value ?? '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9._-]+/g, '-')
|
|
.replace(/-+/g, '-')
|
|
.replace(/^[._-]+|[._-]+$/g, '')
|
|
.slice(0, 80);
|
|
}
|
|
|
|
function extOf(url: string): string {
|
|
const clean = url.split('?')[0] ?? '';
|
|
const ext = clean.split('.').pop()?.toLowerCase() ?? 'webp';
|
|
if (ext === 'webp' || ext === 'png' || ext === 'gif') return ext;
|
|
return 'webp';
|
|
}
|
|
|
|
async function pathExists(p: string): Promise<boolean> {
|
|
try {
|
|
await stat(p);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
interface PetSharePet {
|
|
id: string;
|
|
displayName: string;
|
|
description: string;
|
|
ownerName?: string;
|
|
tags?: string[];
|
|
spritesheetUrl: string;
|
|
spritesheetPath?: string;
|
|
}
|
|
|
|
interface PetShareResponse {
|
|
pets: PetSharePet[];
|
|
page: number;
|
|
pageSize: number;
|
|
total: number;
|
|
totalPages: number;
|
|
}
|
|
|
|
async function listPetSharePets(limit: number | null): Promise<PetTask[]> {
|
|
const tasks: PetTask[] = [];
|
|
let page = 1;
|
|
const pageSize = 24;
|
|
for (;;) {
|
|
const url = `${PETSHARE_BASE}/api/pets?page=${page}&pageSize=${pageSize}`;
|
|
const resp = await fetch(url);
|
|
if (!resp.ok) {
|
|
throw new Error(`petshare list page ${page} failed: ${resp.status} ${resp.statusText}`);
|
|
}
|
|
const data = (await resp.json()) as PetShareResponse;
|
|
for (const pet of data.pets) {
|
|
const folder = sanitizeFolder(pet.id);
|
|
if (!folder) continue;
|
|
const spritesheetUrl = pet.spritesheetUrl.startsWith('http')
|
|
? pet.spritesheetUrl
|
|
: `${PETSHARE_BASE}${pet.spritesheetUrl}`;
|
|
const ext = extOf(pet.spritesheetPath ?? spritesheetUrl);
|
|
tasks.push({
|
|
source: 'petshare',
|
|
folder,
|
|
manifest: {
|
|
id: pet.id,
|
|
displayName: pet.displayName,
|
|
description: pet.description ?? '',
|
|
spritesheetPath: `spritesheet.${ext}`,
|
|
author: pet.ownerName,
|
|
tags: pet.tags ?? [],
|
|
source: 'codex-pet-share',
|
|
sourceUrl: `https://codex-pet-share.pages.dev/#/pets/${encodeURIComponent(pet.id)}`,
|
|
},
|
|
spritesheetUrl,
|
|
spritesheetExt: ext,
|
|
});
|
|
if (limit && tasks.length >= limit) return tasks;
|
|
}
|
|
if (page >= data.totalPages) break;
|
|
page++;
|
|
}
|
|
return tasks;
|
|
}
|
|
|
|
interface HatcheryPet {
|
|
id: string;
|
|
displayName: string;
|
|
description: string;
|
|
petManifestId?: string;
|
|
authorLabel?: string;
|
|
authorXUrl?: string;
|
|
galleryUrl?: string;
|
|
petJsonUrl: string;
|
|
spritesheetUrl: string;
|
|
downloadCount?: number;
|
|
createdAt?: string;
|
|
}
|
|
|
|
interface HatcheryResponse {
|
|
source: string;
|
|
count: number;
|
|
pets: HatcheryPet[];
|
|
}
|
|
|
|
async function listHatcheryPets(limit: number | null): Promise<PetTask[]> {
|
|
const resp = await fetch(HATCHERY_LIST);
|
|
if (!resp.ok) {
|
|
throw new Error(`hatchery list failed: ${resp.status} ${resp.statusText}`);
|
|
}
|
|
const data = (await resp.json()) as HatcheryResponse;
|
|
const tasks: PetTask[] = [];
|
|
for (const pet of data.pets) {
|
|
// Prefer the human-readable manifest id when available — that is
|
|
// what users see in their `~/.codex/pets/` listing.
|
|
const folder = sanitizeFolder(pet.petManifestId || pet.id);
|
|
if (!folder) continue;
|
|
// We will rewrite pet.json from the live `petJsonUrl` content, but
|
|
// also keep our enriched fields so users can trace the origin.
|
|
tasks.push({
|
|
source: 'hatchery',
|
|
folder,
|
|
manifest: {
|
|
id: pet.petManifestId || pet.id,
|
|
displayName: pet.displayName,
|
|
description: pet.description ?? '',
|
|
spritesheetPath: 'spritesheet.webp',
|
|
author: pet.authorLabel,
|
|
authorXUrl: pet.authorXUrl,
|
|
source: 'j20-hatchery',
|
|
sourceUrl: pet.galleryUrl,
|
|
},
|
|
spritesheetUrl: pet.spritesheetUrl,
|
|
spritesheetExt: extOf(pet.spritesheetUrl),
|
|
});
|
|
if (limit && tasks.length >= limit) break;
|
|
}
|
|
return tasks;
|
|
}
|
|
|
|
async function downloadBinary(url: string): Promise<Buffer> {
|
|
const resp = await fetch(url);
|
|
if (!resp.ok) {
|
|
throw new Error(`download ${url} failed: ${resp.status} ${resp.statusText}`);
|
|
}
|
|
const ab = await resp.arrayBuffer();
|
|
return Buffer.from(ab);
|
|
}
|
|
|
|
async function writePet(
|
|
task: PetTask,
|
|
outRoot: string,
|
|
force: boolean,
|
|
): Promise<'wrote' | 'skipped'> {
|
|
const dir = path.join(outRoot, task.folder);
|
|
const sheetPath = path.join(dir, `spritesheet.${task.spritesheetExt}`);
|
|
const manifestPath = path.join(dir, 'pet.json');
|
|
if (!force && (await pathExists(sheetPath)) && (await pathExists(manifestPath))) {
|
|
return 'skipped';
|
|
}
|
|
await mkdir(dir, { recursive: true });
|
|
const bytes = await downloadBinary(task.spritesheetUrl);
|
|
// Validate the magic bytes minimally — abort writes when the server
|
|
// returns an HTML error page (every catalog has had transient hiccups
|
|
// at some point), so callers do not end up with `.webp` files that
|
|
// are actually `<!doctype html>`.
|
|
if (bytes.length < 16) {
|
|
throw new Error(`${task.folder}: spritesheet too small (${bytes.length} bytes)`);
|
|
}
|
|
const head = bytes.subarray(0, 12);
|
|
const isWebp = head.toString('ascii', 0, 4) === 'RIFF' && head.toString('ascii', 8, 12) === 'WEBP';
|
|
const isPng = head.toString('hex', 0, 8) === '89504e470d0a1a0a';
|
|
const isGif = head.toString('ascii', 0, 6) === 'GIF87a' || head.toString('ascii', 0, 6) === 'GIF89a';
|
|
if (!isWebp && !isPng && !isGif) {
|
|
throw new Error(`${task.folder}: spritesheet is not webp/png/gif`);
|
|
}
|
|
await writeFile(sheetPath, bytes);
|
|
await writeFile(manifestPath, JSON.stringify(task.manifest, null, 2) + '\n', 'utf8');
|
|
return 'wrote';
|
|
}
|
|
|
|
async function runPool<T, R>(
|
|
items: T[],
|
|
concurrency: number,
|
|
worker: (item: T, index: number) => Promise<R>,
|
|
): Promise<R[]> {
|
|
const results: R[] = new Array(items.length);
|
|
let cursor = 0;
|
|
const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
for (;;) {
|
|
const idx = cursor++;
|
|
if (idx >= items.length) return;
|
|
results[idx] = await worker(items[idx]!, idx);
|
|
}
|
|
});
|
|
await Promise.all(workers);
|
|
return results;
|
|
}
|
|
|
|
async function main(): Promise<void> {
|
|
let args: Args;
|
|
try {
|
|
args = parseArgs(process.argv);
|
|
} catch (err) {
|
|
console.error(String((err as Error).message ?? err));
|
|
printHelp();
|
|
process.exit(1);
|
|
}
|
|
|
|
if (args.sources.size === 0) {
|
|
console.error('No sources selected — nothing to do.');
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log(`Destination: ${args.out}`);
|
|
await mkdir(args.out, { recursive: true });
|
|
|
|
const tasks: PetTask[] = [];
|
|
if (args.sources.has('petshare')) {
|
|
process.stdout.write('Fetching codex-pet-share catalog…');
|
|
const list = await listPetSharePets(args.limit);
|
|
process.stdout.write(` ${list.length} pets\n`);
|
|
tasks.push(...list);
|
|
}
|
|
if (args.sources.has('hatchery')) {
|
|
process.stdout.write('Fetching j20 hatchery catalog…');
|
|
const list = await listHatcheryPets(args.limit);
|
|
process.stdout.write(` ${list.length} pets\n`);
|
|
tasks.push(...list);
|
|
}
|
|
|
|
if (tasks.length === 0) {
|
|
console.log('No pets to download.');
|
|
return;
|
|
}
|
|
|
|
// Earlier sources win when two catalogs publish the same folder name
|
|
// (e.g. an upstream "goku" appears in both feeds). De-duplicate so we
|
|
// do not race two writers on the same folder.
|
|
const dedup = new Map<string, PetTask>();
|
|
for (const task of tasks) {
|
|
if (!dedup.has(task.folder)) dedup.set(task.folder, task);
|
|
}
|
|
const unique = Array.from(dedup.values());
|
|
|
|
let wrote = 0;
|
|
let skipped = 0;
|
|
let failed = 0;
|
|
await runPool(unique, args.concurrency, async (task) => {
|
|
try {
|
|
const result = await writePet(task, args.out, args.force);
|
|
if (result === 'wrote') {
|
|
wrote++;
|
|
console.log(`+ ${task.source.padEnd(8)} ${task.folder}`);
|
|
} else {
|
|
skipped++;
|
|
}
|
|
} catch (err) {
|
|
failed++;
|
|
console.error(`! ${task.source.padEnd(8)} ${task.folder}: ${(err as Error).message}`);
|
|
}
|
|
});
|
|
|
|
console.log(`\nDone. wrote=${wrote} skipped=${skipped} failed=${failed} (total=${unique.length})`);
|
|
if (failed > 0) process.exitCode = 1;
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|