Add map import and export with durable refresh progress

This commit is contained in:
2026-10-07 19:44:49 +05:00
parent dac8650ac1
commit 1307436e65
37 changed files with 2207 additions and 85 deletions

View File

@@ -19,6 +19,13 @@ import { multiplayerApi } from "./multiplayer-api";
import { contributionCutoff, reportSchema } from "../shared/moderation";
import { siteSettingsSchema } from "../shared/site-settings";
import { getSiteSettings } from "./site-settings";
import { processUploadAfterResponse } from "./process-uploads";
import { needsProcessing } from "./fabric-jobs";
import { archiveMetadataSchema, importManifest } from "../shared/map-archive";
import {
archiveSignature,
verifyArchiveSignature,
} from "./map-archive-signature";
const hash = (value: string) =>
createHash("sha256").update(value).digest("hex");
const idSchema = z.string().uuid();
@@ -259,20 +266,39 @@ export async function handle(request: NextRequest, path: string[]) {
}
if (path[0] === "uploads") {
const owner = await contributor(request);
if (method === "POST" && path.length === 1) {
if (
method === "POST" &&
(path.length === 1 || (path[1] === "import" && path.length === 2))
) {
await limit("upload:" + owner, 10, 3600);
const manifest = parseUploadManifest(
(await jsonBody(request)).manifest,
);
const body = await jsonBody(request);
const manifest =
path[1] === "import"
? importManifest(body.manifest)
: parseUploadManifest(body.manifest);
const archive =
path[1] === "import"
? archiveMetadataSchema.parse(
(({ assets: _assets, ...metadata }) => metadata)(body.manifest),
)
: null;
if (archive)
verifyArchiveSignature(archive, process.env.SUPABASE_SECRET_KEY!);
if (
manifest.assets.reduce((n, a) => n + a.bytes, 0) >
Number(process.env.MINEGEO_MAX_UPLOAD_BYTES || 268435456)
)
throw new Error("Dataset exceeds upload limit");
const datasetId = required(
await client.rpc("create_upload", {
await client.rpc(archive ? "create_map_import" : "create_upload", {
p_owner: owner,
p_manifest: manifest,
...(archive
? {
p_original_map: archive.originalMapId,
p_original_owner: archive.originalOwner,
}
: {}),
}),
);
return NextResponse.json({ datasetId });
@@ -333,6 +359,7 @@ export async function handle(request: NextRequest, path: string[]) {
p_owner: owner,
}),
);
if (state === "processing") processUploadAfterResponse(d.id);
return NextResponse.json({ datasetId: d.id, state });
}
}
@@ -343,7 +370,7 @@ export async function handle(request: NextRequest, path: string[]) {
await client
.from("datasets")
.select(
"*, upload_jobs!dataset_id(state,progress,error,updated_at,lease_until,kind)",
"*, upload_jobs!dataset_id(id,state,progress,error,updated_at,lease_until,kind,source_dataset_id)",
)
.eq("owner_id", owner)
.or(
@@ -353,15 +380,87 @@ export async function handle(request: NextRequest, path: string[]) {
);
return NextResponse.json({
maps: await Promise.all(
maps.map(async (m) => ({
...m,
image: await mediaUrl(m.cover_path),
map: await mediaUrl(m.map_path),
})),
maps.map(async (m) => {
if (needsProcessing(m.upload_jobs))
processUploadAfterResponse(m.id);
return {
...m,
image: await mediaUrl(m.cover_path),
map: await mediaUrl(m.map_path),
};
}),
),
});
}
const d = await ownedDataset(request, path[1]);
if (method === "GET" && path[2] === "export" && path.length === 3) {
if (
d.imported_from_id ||
d.parent_dataset_id ||
d.banned_at ||
!d.map_path ||
!["draft", "rejected", "review", "published", "unpublished"].includes(
d.state,
)
)
throw new Error("This map is not ready to export");
await limit("export:" + d.owner_id, 10, 3600);
const input = required(
await client
.from("dataset_inputs")
.select("manifest,validated")
.eq("dataset_id", d.id)
.single(),
);
if (!input.validated)
throw new Error("This map is not ready to export");
const manifest = datasetSchema.parse(input.manifest);
const profile = required(
await client
.from("profiles")
.select("display_name")
.eq("id", d.owner_id)
.single(),
);
const metadata = archiveMetadataSchema.parse({
format: "minegeo-map/v1",
minecraftVersion: manifest.minecraftVersion,
dimension: manifest.dimension,
north: manifest.north,
source: manifest.source,
name: manifest.name,
description: manifest.description,
serverName: manifest.serverName,
bounds: manifest.bounds,
viewpoints: manifest.viewpoints.map(({ id, x, y, z }) => ({
id,
x,
y,
z,
})),
originalMapId: d.id,
originalOwner: profile.display_name || "Explorer",
signature: "0".repeat(64),
});
metadata.signature = archiveSignature(
metadata,
process.env.SUPABASE_SECRET_KEY!,
);
const assets = [
{ path: "map.webp", storage: manifest.mapPath ?? d.map_path },
...manifest.viewpoints.flatMap((p) =>
faces.map((f) => ({
path: `${p.id}/${f}.webp`,
storage: panoramaPath(d.id, p, `${f}.webp`),
})),
),
];
const urls = await mediaUrls(assets.map((a) => a.storage));
return NextResponse.json({
manifest: metadata,
assets: assets.map((a, i) => ({ path: a.path, url: urls[i] })),
});
}
if (path[2] === "appeals" && path.length === 3 && !d.parent_dataset_id) {
if (method === "GET") {
const appeals = required(
@@ -405,6 +504,7 @@ export async function handle(request: NextRequest, path: string[]) {
})
).error,
);
processUploadAfterResponse(d.id);
return NextResponse.json({ ok: true });
}
if (method === "GET") {
@@ -466,10 +566,13 @@ export async function handle(request: NextRequest, path: string[]) {
const manifest = datasetSchema.parse(input.manifest);
const jobResult = await client
.from("upload_jobs")
.select("state,progress,error,updated_at,lease_until,kind")
.select(
"id,state,progress,error,updated_at,lease_until,kind,source_dataset_id",
)
.eq("dataset_id", d.id)
.maybeSingle();
checkServiceError(jobResult.error);
if (needsProcessing(jobResult.data)) processUploadAfterResponse(d.id);
return NextResponse.json({
...d,
upload_jobs: jobResult.data,
@@ -540,6 +643,11 @@ export async function handle(request: NextRequest, path: string[]) {
return NextResponse.json({ ok: true });
}
if (method === "POST" && path[2] === "submit") {
if (d.imported_from_id)
throw new ServiceError(
`Owned by "${d.imported_owner_name}". Shared maps cannot be submitted for moderator review.`,
403,
);
const { error } = await client.rpc("submit_draft", {
p_dataset: d.id,
p_owner: d.owner_id,
@@ -794,12 +902,18 @@ export async function handle(request: NextRequest, path: string[]) {
const reviewer = await moderator(request);
if (path[1] === "settings" && path.length === 2 && method === "PATCH") {
const body = siteSettingsSchema.parse(await jsonBody(request));
const enabled = required(await client.rpc("set_beta_badge", {
p_reviewer: reviewer, p_enabled: body.betaBadgeEnabled,
}));
return NextResponse.json({ betaBadgeEnabled: enabled }, {
headers: { "Cache-Control": "no-store" },
});
const enabled = required(
await client.rpc("set_beta_badge", {
p_reviewer: reviewer,
p_enabled: body.betaBadgeEnabled,
}),
);
return NextResponse.json(
{ betaBadgeEnabled: enabled },
{
headers: { "Cache-Control": "no-store" },
},
);
}
if (path[1] === "appeals" && path.length === 3 && method === "POST") {
const body = z

98
lib/fabric-images.ts Normal file
View File

@@ -0,0 +1,98 @@
import { createHash } from "node:crypto";
import sharp from "sharp";
import { faces, type DatasetManifest } from "../shared/protocol";
type ImageIO = {
download: (path: string) => Promise<Buffer>;
save: (path: string, bytes: Buffer) => Promise<void>;
progress: (percent: number) => Promise<void>;
signal: AbortSignal;
mapOnly?: boolean;
};
// Process one image at a time. Vercel needs neither a local capture directory
// nor a Minecraft/Chromium process for images already captured by the mod.
export async function processFabricImages(
manifest: DatasetManifest,
io: ImageIO,
) {
if (manifest.source !== "fabric" && !manifest.mediaFormat)
throw new Error("A Fabric capture is required");
const format = manifest.mediaFormat ?? "png";
const assets = new Map(manifest.assets.map((asset) => [asset.path, asset]));
const points = io.mapOnly
? []
: (manifest.availableViewpoints ?? manifest.viewpoints);
const total = 1 + points.length * faces.length;
let completed = 0;
async function read(path: string) {
io.signal.throwIfAborted();
const asset = assets.get(path);
if (!asset) throw new Error(`Missing export asset: ${path}`);
const bytes = await io.download(path);
io.signal.throwIfAborted();
if (
bytes.length !== asset.bytes ||
createHash("sha256").update(bytes).digest("hex") !== asset.sha256
)
throw new Error(`Asset checksum/size mismatch: ${path}`);
return bytes;
}
async function save(path: string, bytes: Buffer) {
io.signal.throwIfAborted();
await io.save(path, bytes);
io.signal.throwIfAborted();
}
const map = await read(`map.${format}`);
const mapMeta = await sharp(map, { limitInputPixels: 16777216 }).metadata();
if (
mapMeta.format !== format ||
!mapMeta.width ||
!mapMeta.height ||
(mapMeta.pages ?? 1) !== 1
)
throw new Error("The captured map has an invalid image format");
await save(
"map.webp",
await sharp(map, { limitInputPixels: 16777216 })
.webp({ lossless: true })
.toBuffer(),
);
await io.progress((95 * ++completed) / total);
for (const point of points) {
for (const face of faces) {
const bytes = await read(`${point.id}/${face}.${format}`);
const meta = await sharp(bytes, {
limitInputPixels: 16777216,
}).metadata();
if (
meta.format !== format ||
(meta.pages ?? 1) !== 1 ||
!meta.width ||
meta.width !== meta.height ||
meta.width < 256 ||
meta.width > 2048
)
throw new Error(
format === "png"
? "Panorama faces must be square PNGs, 256–2048 pixels"
: "Panorama faces must be square WebPs, 256–2048 pixels",
);
await save(
`${point.id}/${face}.webp`,
await sharp(bytes, { limitInputPixels: 16777216 })
.webp({ quality: 85 })
.toBuffer(),
);
if (face === "nz")
await save(
`${point.id}/preview.webp`,
await sharp(bytes, { limitInputPixels: 16777216 })
.resize(512, 512)
.webp({ quality: 76 })
.toBuffer(),
);
await io.progress((95 * ++completed) / total);
}
}
}

198
lib/fabric-jobs.ts Normal file
View File

@@ -0,0 +1,198 @@
import { randomUUID } from "node:crypto";
import type { SupabaseClient } from "@supabase/supabase-js";
import type { Database } from "../shared/database";
import { datasetSchema } from "../shared/protocol";
import { processFabricImages } from "./fabric-images";
type Job = Database["public"]["Tables"]["upload_jobs"]["Row"];
type JobStatus = Pick<Job, "state" | "lease_until">;
export function needsProcessing(job: JobStatus | null | undefined) {
return (
!!job &&
(job.state === "queued" ||
(job.state === "running" &&
!!job.lease_until &&
Date.parse(job.lease_until) < Date.now()))
);
}
function checked<T>(result: {
data: T;
error: { message: string } | null;
}): NonNullable<T> {
if (result.error) throw new Error(result.error.message);
if (result.data == null) throw new Error("Processing record not found");
return result.data!;
}
// Same queue, attempts and two-minute renewable leases as the standalone worker.
// Scope each invocation to a dataset the API has already authorized.
export async function processFabricJob(
client: SupabaseClient<Database>,
datasetId: string,
deadlineAt = Date.now() + 250000,
) {
const pending = checked(
await client
.from("upload_jobs")
.select("*")
.eq("dataset_id", datasetId)
.maybeSingle(),
);
if (
!needsProcessing(pending) ||
pending.attempts >= 3 ||
deadlineAt - Date.now() < 20000
)
return "skipped";
const mapOnly = pending.kind === "map";
const sourceId = mapOnly ? pending.source_dataset_id : pending.dataset_id;
if (!sourceId) throw new Error("Missing capture for map refresh");
const dataset = checked(
await client
.from("datasets")
.select("owner_id,source")
.eq("id", sourceId)
.single(),
);
// Terrain exports still use the standalone renderer; do not consume a retry.
const input = checked(
await client
.from("dataset_inputs")
.select("manifest")
.eq("dataset_id", sourceId)
.single(),
);
const manifest = datasetSchema.parse(input.manifest);
if (dataset.source !== "fabric" && !manifest.mediaFormat) return "skipped";
const worker = "vercel-" + randomUUID();
const now = Date.now();
let claim = client
.from("upload_jobs")
.update({
state: "running",
attempts: pending.attempts + 1,
lease_owner: worker,
lease_until: new Date(now + 120000).toISOString(),
progress: 0,
error: null,
updated_at: new Date(now).toISOString(),
})
.eq("id", pending.id)
.eq("state", pending.state)
.eq("attempts", pending.attempts);
if (pending.state === "running")
claim = claim.lt("lease_until", new Date(now).toISOString());
const rows = checked(await claim.select("*"));
if (!rows.length) return "skipped";
const job = rows[0];
const abort = new AbortController();
// Leave time to release the lease before Vercel's 300-second limit.
const deadline = setTimeout(
() =>
abort.abort(new Error("Image processing time limit reached; retrying")),
Math.max(1, deadlineAt - Date.now()),
);
let lastProgress = 0;
async function update(
values: Database["public"]["Tables"]["upload_jobs"]["Update"],
) {
const rows = checked(
await client
.from("upload_jobs")
.update(values)
.eq("id", job.id)
.eq("lease_owner", worker)
.eq("state", "running")
.select("id"),
);
if (!rows.length) abort.abort(new Error("Lease lost or cancelled"));
abort.signal.throwIfAborted();
}
const heartbeat = setInterval(() => {
void update({
lease_until: new Date(Date.now() + 120000).toISOString(),
updated_at: new Date().toISOString(),
}).catch((error) => abort.abort(error));
}, 20000);
try {
console.log(
`Vercel worker: processing ${job.id} (${manifest.viewpoints.length} panoramas)`,
);
await processFabricImages(manifest, {
mapOnly,
signal: abort.signal,
async download(path) {
const blob = checked(
await client.storage
.from("minegeo-raw")
.download(`${dataset.owner_id}/${sourceId}/${path}`),
);
return Buffer.from(await blob.arrayBuffer());
},
async save(path, bytes) {
checked(
await client.storage
.from("minegeo-media")
.upload(
mapOnly
? `${job.dataset_id}/maps/${job.id}.webp`
: `${job.dataset_id}/${path}`,
bytes,
{ contentType: "image/webp", upsert: true, cacheControl: "60" },
),
);
},
async progress(percent) {
if (Date.now() - lastProgress < 2000 && percent < 95) return;
lastProgress = Date.now();
await update({
progress: Math.floor(percent),
updated_at: new Date().toISOString(),
});
},
});
abort.signal.throwIfAborted();
const result = await client.rpc("complete_job", {
p_job: job.id,
p_worker: worker,
p_count: manifest.viewpoints.length,
p_cover: manifest.viewpoints[0].id,
});
if (result.error) throw new Error(result.error.message);
console.log(`Vercel worker: ${job.id} complete; private draft ready`);
return "complete";
} catch (error) {
const message =
error instanceof Error ? error.message : "Image processing failed";
console.error(`Vercel worker: ${job.id}: ${message}`);
const state = job.attempts >= 3 ? "failed" : "queued";
const released = checked(
await client
.from("upload_jobs")
.update({
state,
error: message,
lease_owner: null,
lease_until: null,
progress: 0,
updated_at: new Date().toISOString(),
})
.eq("id", job.id)
.eq("lease_owner", worker)
.eq("state", "running")
.select("id"),
);
if (!mapOnly && state === "failed" && released.length) {
const result = await client
.from("datasets")
.update({ state: "failed", rejection_reason: message })
.eq("id", job.dataset_id)
.eq("state", "processing");
if (result.error) throw new Error(result.error.message);
}
return released.length ? state : "skipped";
} finally {
clearInterval(heartbeat);
clearTimeout(deadline);
}
}

View File

@@ -0,0 +1,23 @@
import { createHmac, timingSafeEqual } from "node:crypto";
import {
archiveMetadataSchema,
type ArchiveMetadata,
} from "../shared/map-archive";
export function archiveSignature(value: ArchiveMetadata, key: string) {
const { signature: _signature, ...metadata } =
archiveMetadataSchema.strip().parse(value);
return createHmac("sha256", key)
.update("minegeo-map-share/v1\n")
.update(JSON.stringify(metadata))
.digest("hex");
}
export function verifyArchiveSignature(value: ArchiveMetadata, key: string) {
const expected = Buffer.from(archiveSignature(value, key), "hex");
const supplied = Buffer.from(value.signature, "hex");
if (
expected.length !== supplied.length ||
!timingSafeEqual(expected, supplied)
)
throw new Error("Map ZIP ownership verification failed");
}

133
lib/map-transfer.ts Normal file
View File

@@ -0,0 +1,133 @@
import { api } from "./client-api";
import type { ArchiveMetadata } from "../shared/map-archive";
type Progress = (message: string, values?: Record<string, number>) => void;
export async function exportMap(id: string, progress: Progress) {
const {
archiveMetadataSchema,
archiveSchema,
archiveLimit,
archiveFileLimit,
archiveManifestName,
sha256,
} = await import("../shared/map-archive");
const { zipSync, strToU8 } = await import("fflate");
const result = await api<{
manifest: ArchiveMetadata;
assets: { path: string; url: string }[];
}>(`datasets/${id}/export`);
const metadata = archiveMetadataSchema.parse(result.manifest);
const files: Record<string, Uint8Array> = Object.create(null);
const assets = [];
let total = 0;
for (const [index, asset] of result.assets.entries()) {
progress("Exporting images: {done} / {total}", {
done: index + 1,
total: result.assets.length,
});
const response = await fetch(asset.url, {
signal: AbortSignal.timeout(60000),
cache: "no-store",
});
if (!response.ok || !response.body)
throw new Error("Could not download map images. Please try again.");
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
while (true) {
const { value, done } = await reader.read();
if (done) break;
size += value.length;
total += value.length;
if (size > archiveFileLimit || total > archiveLimit - 128 * 1024) {
await reader.cancel();
throw new Error("Map archive exceeds the 256 MB limit");
}
chunks.push(value);
}
const bytes = new Uint8Array(size);
let offset = 0;
for (const chunk of chunks) {
bytes.set(chunk, offset);
offset += chunk.length;
}
files[asset.path] = bytes;
assets.push({ path: asset.path, bytes: size, sha256: await sha256(bytes) });
}
const archive = archiveSchema.parse({ ...metadata, assets });
files[archiveManifestName] = strToU8(JSON.stringify(archive));
progress("Preparing ZIP…");
const zip = zipSync(files, { level: 0 });
if (zip.length > archiveLimit)
throw new Error("Map archive exceeds the 256 MB limit");
const url = URL.createObjectURL(
new Blob([new Uint8Array(zip).buffer], { type: "application/zip" }),
);
const link = document.createElement("a");
link.href = url;
link.download = `${metadata.name.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 80) || "minegeo-map"}.zip`;
document.body.appendChild(link);
link.click();
link.remove();
setTimeout(() => URL.revokeObjectURL(url), 60000);
}
export async function importMap(file: File, progress: Progress) {
const { archiveLimit, readMapArchive } = await import(
"../shared/map-archive"
);
if (!file.name.toLowerCase().endsWith(".zip"))
throw new Error("Choose a MineGeo ZIP file");
if (file.size > archiveLimit)
throw new Error("Map archive exceeds the 256 MB limit");
progress("Checking map ZIP…");
const { archive, files } = await readMapArchive(
new Uint8Array(await file.arrayBuffer()),
);
const { datasetId } = await api<{ datasetId: string }>("uploads/import", {
manifest: archive,
});
const signed = await api<{
assets: {
path: string;
objectName: string;
token: string;
apiKey: string;
endpoint: string;
}[];
}>("uploads/sign", { datasetId });
const { createBrowserClient } = await import("@supabase/ssr");
const client = createBrowserClient(
process.env.NEXT_PUBLIC_SUPABASE_URL!,
process.env.NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY!,
);
try {
for (const [index, asset] of signed.assets.entries()) {
progress("Importing images: {done} / {total}", {
done: index + 1,
total: signed.assets.length,
});
const content = files.get(asset.path);
if (!content) throw new Error("Invalid or incomplete map ZIP");
const { error } = await client.storage
.from("minegeo-raw")
.uploadToSignedUrl(
asset.objectName,
asset.token,
new Blob([new Uint8Array(content).buffer], { type: "image/webp" }),
{ contentType: "image/webp" },
);
if (error)
throw new Error(
"Map upload failed. Check your connection and import the ZIP again.",
);
}
progress("Preparing your private draft…");
await api("uploads/finalize", { datasetId });
return datasetId;
} catch (error) {
await api(`datasets/${datasetId}/cancel`, {}).catch(() => {});
throw error;
}
}

View File

@@ -2,7 +2,7 @@ import "server-only";
import { createHash, randomBytes } from "node:crypto";
import { NextRequest, NextResponse } from "next/server";
import { z } from "zod";
import { admin, required } from "./supabase";
import { admin, required, user } from "./supabase";
import { mediaUrls } from "./catalog";
import { requestOrigin } from "./request-origin";
import { ServiceError } from "./service-error";
@@ -53,13 +53,15 @@ export async function multiplayerApi(
if (request.method === "POST" && path.length === 1) {
const body = createRoomSchema.parse(await jsonBody(request));
await limit("multiplayer:create:" + accessHash, 20, 3600);
const account = await user();
const id = required(
await client.rpc("create_multiplayer_room", {
await client.rpc("create_map_room", {
p_dataset: body.datasetId,
p_difficulty: body.difficulty,
p_name: body.displayName,
p_limit: body.playerLimit,
p_access_hash: accessHash,
p_owner: account?.id ?? null,
}),
);
return respond({ id });

22
lib/process-uploads.ts Normal file
View File

@@ -0,0 +1,22 @@
import "server-only";
import { after } from "next/server";
import { admin } from "./supabase";
import { processFabricJob } from "./fabric-jobs";
export function processUploadAfterResponse(datasetId: string) {
after(async () => {
try {
const deadlineAt = Date.now() + 250000;
for (let attempt = 0; attempt < 3; attempt++) {
const state = await processFabricJob(admin(), datasetId, deadlineAt);
if (state !== "queued" || deadlineAt - Date.now() < 20000) break;
await new Promise((resolve) => setTimeout(resolve, 2500));
}
} catch (error) {
console.error(
"Vercel upload worker:",
error instanceof Error ? error.message : error,
);
}
});
}