Report renderer progress and detect stalled jobs

.env.example

@@ -15,4 +15,6 @@ DATABASE_URL=
 MINEGEO_FACE_SIZE=1024
 # Worker heap ceiling in MiB; does not reserve memory or change the web server heap.
 MINEGEO_WORKER_HEAP_MB=4096
+# Fail with an actionable error if rendering makes no reported progress for this long.
+MINEGEO_RENDER_IDLE_SECONDS=600
 MINEGEO_MAX_UPLOAD_BYTES=268435456

app/block-theme.css

@@ -916,3 +916,26 @@ h3 {
     scroll-behavior: auto !important;
   }
 }
+.processing-progress {
+  margin: 16px 0;
+  max-width: 640px;
+}
+.processing-progress p {
+  margin: 8px 0;
+}
+.processing-progress progress {
+  width: 100%;
+  height: 16px;
+  accent-color: #81c64b;
+  border: 2px solid #273529;
+  background: #101720;
+}
+.processing-progress progress::-webkit-progress-bar {
+  background: #101720;
+}
+.processing-progress progress::-webkit-progress-value {
+  background: #81c64b;
+}
+.processing-progress progress::-moz-progress-bar {
+  background: #81c64b;
+}

components/Dashboard.tsx

@@ -5,8 +5,10 @@ import { createBrowserClient } from "@supabase/ssr";
 import { useRouter } from "next/navigation";
 import { api } from "./StartGame";
 import type { MapListing } from "@/lib/catalog";
+import type { ProcessingJob } from "@/shared/processing-progress";
+import { ProcessingProgress } from "./ProcessingProgress";
 type Draft = MapListing & {
-  upload_jobs: { state: string; progress: number; error: string | null } | null;
+  upload_jobs: ProcessingJob | null;
 };
 export function Dashboard() {
   const [maps, setMaps] = useState<Draft[]>([]),
@@ -101,12 +103,8 @@ export function Dashboard() {
             </span>
             <h3>{m.name}</h3>
             <span className="badge">{m.state}</span>{" "}
-            <span className="muted">
+            <span className="muted">{m.panorama_count} locations</span>
-              {m.panorama_count} locations{" "}
+            <ProcessingProgress job={m.upload_jobs} />
-              {m.upload_jobs?.state === "running"
-                ? `· ${m.upload_jobs.progress}% processed`
-                : ""}
-            </span>
             {m.rejection_reason && (
               <p className="error-message">{m.rejection_reason}</p>
             )}

components/DraftEditor.tsx

@@ -3,6 +3,8 @@ import { useEffect, useState } from "react";
 import Image from "next/image";
 import { api } from "./StartGame";
 import type { DatasetManifest } from "@/shared/protocol";
+import type { ProcessingJob } from "@/shared/processing-progress";
+import { ProcessingProgress } from "./ProcessingProgress";
 type Draft = {
   id: string;
   name: string;
@@ -13,6 +15,7 @@ type Draft = {
   map: string | null;
   manifest: DatasetManifest;
   previews: { id: string; image: string | null }[];
+  upload_jobs: ProcessingJob | null;
 };
 export function DraftEditor({ id }: { id: string }) {
   const [draft, setDraft] = useState<Draft | null>(null),
@@ -29,6 +32,7 @@ export function DraftEditor({ id }: { id: string }) {
       api<Draft>("datasets/" + id)
         .then((d) => {
           if (disposed) return;
+          setError("");
           setDraft(d);
           setName(d.name);
           setDescription(d.description);
@@ -37,7 +41,10 @@ export function DraftEditor({ id }: { id: string }) {
             timer = setTimeout(load, 5000);
         })
         .catch((e) => {
-          if (!disposed) setError(e.message);
+          if (!disposed) {
+            setError(e.message);
+            timer = setTimeout(load, 5000);
+          }
         });
     load();
     return () => {
@@ -89,6 +96,7 @@ export function DraftEditor({ id }: { id: string }) {
   return (
     <>
       <span className="badge">{draft.state}</span>
+      <ProcessingProgress job={draft.upload_jobs} />
       {["uploading", "processing"].includes(draft.state) && (
         <p role="status" className="success-message">
           {draft.state === "uploading"

components/ProcessingProgress.tsx

@@ -0,0 +1,42 @@
+"use client";
+import { useEffect, useState } from "react";
+import {
+  processingStage,
+  type ProcessingJob,
+} from "@/shared/processing-progress";
+
+export function ProcessingProgress({ job }: { job: ProcessingJob | null }) {
+  const [now, setNow] = useState<number | null>(null);
+  useEffect(() => {
+    setNow(Date.now());
+    const timer = setInterval(() => setNow(Date.now()), 5000);
+    return () => clearInterval(timer);
+  }, []);
+  if (!job || !["queued", "running"].includes(job.state)) return null;
+  const progress = Math.max(0, Math.min(100, job.progress));
+  const stopped =
+    now !== null &&
+    job.state === "running" &&
+    (!job.lease_until || Date.parse(job.lease_until) < now);
+  return (
+    <div className="processing-progress" role="status" aria-live="polite">
+      <p>
+        {job.state === "queued"
+          ? "Waiting for a processing worker"
+          : `${progress}% · ${processingStage(progress)}`}
+      </p>
+      <progress
+        max={100}
+        value={progress}
+        aria-label="Map processing progress"
+      />
+      <p className="muted">
+        {stopped
+          ? "The worker has stopped reporting. Check its terminal and restart it if necessary. Your uploaded export is saved."
+          : job.state === "running"
+            ? "Worker connected. Large exports can take a while; progress measures completed work."
+            : "Processing will start when a worker is available."}
+      </p>
+    </div>
+  );
+}

docs/server-plugin.md

@@ -48,6 +48,10 @@ Upload sends the latest **completed** export. It does not generate an export. Th
 
 The worker renders the map and panoramas. Open the private draft in your dashboard; the page updates while processing. Inspect the result, choose at least five locations, and submit for moderation. Once a moderator approves it, players can select it in the catalog.
 
+Both the dashboard and draft show completed-work progress: downloading, reading/indexing terrain, loading models, building geometry, rendering panorama faces, rendering the surface map, and saving images. Percentages are not a time estimate. The terminal reports block/face counts as work advances. A large export can take longer on a Mac using software rendering.
+
+If the worker stops renewing its lease, the page warns that it has stopped reporting. Renderer cancellation closes its browser, and `MINEGEO_RENDER_IDLE_SECONDS` (default 600) fails a renderer that reports no progress for that period. This is an inactivity limit, not a limit on total rendering time. Use a smaller export area or lower `MINEGEO_FACE_SIZE` if rendering one face exceeds it.
+
 ## Retry, cancel, and configure
 
 - An interrupted transfer resumes with `minegeo upload`. Progress is saved in `.upload-state.json` beside the export. Already received exports are not submitted twice.

lib/api.ts

@@ -338,7 +338,9 @@ export async function handle(request: NextRequest, path: string[]) {
         const maps = required(
           await client
             .from("datasets")
-            .select("*, upload_jobs(state,progress,error)")
+            .select(
+              "*, upload_jobs(state,progress,error,updated_at,lease_until)",
+            )
             .eq("owner_id", owner)
             .order("created_at", { ascending: false }),
         );
@@ -362,8 +364,15 @@ export async function handle(request: NextRequest, path: string[]) {
             .single(),
         );
         const manifest = datasetSchema.parse(input.manifest);
+        const jobResult = await client
+          .from("upload_jobs")
+          .select("state,progress,error,updated_at,lease_until")
+          .eq("dataset_id", d.id)
+          .maybeSingle();
+        if (jobResult.error) throw new Error(jobResult.error.message);
         return NextResponse.json({
           ...d,
+          upload_jobs: jobResult.data,
           manifest,
           map: await mediaUrl(d.map_path),
           previews: await Promise.all(

renderer/render.mjs

@@ -7,12 +7,12 @@ import { assetsRoot } from "../scripts/assets.mjs";
 /** @param {import('../shared/protocol').TerrainSnapshot} snapshot
  * @param {Array<{id:string,x:number,y:number,z:number}>} viewpoints
  * @param {string} output
- * @param {{size?:number,signal?:AbortSignal}} [options] */
+ * @param {{size?:number,signal?:AbortSignal,onProgress?:(event:{percent:number,message:string})=>Promise<void>}} [options] */
 export async function renderSnapshot(
   snapshot,
   viewpoints,
   output,
-  { size = 1024, signal } = {},
+  { size = 1024, signal, onProgress } = {},
 ) {
   const start = Date.now();
   await mkdir(output, { recursive: true });
@@ -26,14 +26,17 @@ export async function renderSnapshot(
         const send = async (chunk) => {
           if (res.destroyed) throw new Error("Terrain connection closed");
           await new Promise((resolve, reject) => {
-            res.write(chunk, (error) => error ? reject(error) : resolve());
+            res.write(chunk, (error) => (error ? reject(error) : resolve()));
           });
         };
         const { blocks, ...metadata } = snapshot;
         await send(JSON.stringify(metadata).slice(0, -1) + ',"blocks":[');
         for (let i = 0; i < blocks.length; i += 2048) {
           if (signal?.aborted) throw new Error("Rendering cancelled");
-          await send((i ? "," : "") + JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1));
+          await send(
+            (i ? "," : "") +
+              JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1),
+          );
         }
         res.end("]}");
         return;
@@ -63,19 +66,50 @@ export async function renderSnapshot(
       );
       res.end(await readFile(path));
     } catch {
-      if (res.headersSent) { res.destroy(); return; }
+      if (res.headersSent) {
+        res.destroy();
+        return;
+      }
       res.writeHead(404);
       res.end();
     }
   });
   await new Promise((r) => server.listen(0, "127.0.0.1", r));
   let browser;
+  let lastActivity = Date.now();
+  let failure;
+  const report = async (event) => {
+    lastActivity = Date.now();
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await onProgress?.(event);
+  };
+  const cancel = () => {
+    failure = new Error("Rendering cancelled or worker lease lost");
+    void browser?.close().catch(() => {});
+  };
+  const idleSeconds = Number(process.env.MINEGEO_RENDER_IDLE_SECONDS || 600);
+  if (!Number.isFinite(idleSeconds) || idleSeconds < 60) {
+    await new Promise((r) => server.close(r));
+    throw new Error("MINEGEO_RENDER_IDLE_SECONDS must be at least 60");
+  }
+  const watchdog = setInterval(() => {
+    if (Date.now() - lastActivity > idleSeconds * 1000) {
+      failure = new Error(
+        `Renderer stopped reporting progress for ${idleSeconds} seconds. Try a smaller export area or lower MINEGEO_FACE_SIZE.`,
+      );
+      void browser?.close().catch(() => {});
+    }
+  }, 10000);
+  signal?.addEventListener("abort", cancel, { once: true });
   try {
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await report({ percent: 24, message: "Starting terrain renderer" });
     browser = await chromium.launch({
       headless: true,
       args: ["--use-angle=swiftshader", "--enable-unsafe-swiftshader"],
     });
     const page = await browser.newPage();
+    await page.exposeFunction("minegeoProgress", report);
     page.on("pageerror", (err) => console.error("Renderer:", err.message));
     await page.goto(`http://127.0.0.1:${server.address().port}`);
     await page.waitForFunction(() => window.rendererReady);
@@ -84,6 +118,11 @@ export async function renderSnapshot(
       if (!response.ok) throw new Error("Unable to load terrain snapshot");
       return window.buildTerrain(await response.json());
     });
+    await report({
+      percent: 60,
+      message: `Terrain ready: ${stats.blocks} blocks; rendering ${viewpoints.length} panoramas`,
+    });
+    let completedFaces = 0;
     for (const p of viewpoints) {
       if (signal?.aborted) throw new Error("Rendering cancelled");
       await mkdir(`${output}/${p.id}`, { recursive: true });
@@ -101,19 +140,32 @@ export async function renderSnapshot(
             .resize(512, 512)
             .webp({ quality: 76 })
             .toFile(`${output}/${p.id}/preview.webp`);
+        await report({
+          percent: 60 + (25 * ++completedFaces) / (viewpoints.length * 6),
+          message: `Panorama faces ${completedFaces}/${viewpoints.length * 6}`,
+        });
       }
     }
+    await report({ percent: 85, message: "Rendering the surface map" });
     const map = await page.evaluate(() => window.renderMap(1024));
     await sharp(Buffer.from(map.split(",")[1], "base64"))
       .webp({ quality: 90 })
       .toFile(`${output}/map.webp`);
+    await report({
+      percent: 90,
+      message: "Rendered all panoramas and the surface map",
+    });
     stats.renderMs = Date.now() - start;
     await writeFile(
       `${output}/render-report.json`,
       JSON.stringify(stats, null, 2),
     );
     return stats;
+  } catch (error) {
+    throw failure || error;
   } finally {
+    clearInterval(watchdog);
+    signal?.removeEventListener("abort", cancel);
     await browser?.close();
     await new Promise((r) => server.close(r));
   }

renderer/scene.js

@@ -162,9 +162,28 @@ window.buildTerrain = async (snapshot) => {
     unsupported = new Set();
   const occupancy = new Map();
   const distinctStates = new Set();
-  for (const b of snapshot.blocks) {
+  let lastProgress = 0;
+  async function progress(percent, message, force = false) {
+    if (!force && performance.now() - lastProgress < 3000) return;
+    lastProgress = performance.now();
+    // Yield to browser tasks so bindings, cancellation and timers can run.
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    await window.minegeoProgress?.({ percent, message });
+  }
+  await progress(
+    25,
+    `Indexing ${snapshot.blocks.length.toLocaleString()} terrain blocks`,
+    true,
+  );
+  for (let i = 0; i < snapshot.blocks.length; i++) {
+    const b = snapshot.blocks[i];
     occupancy.set(`${b[0]},${b[1]},${b[2]}`, b[3]);
     distinctStates.add(b[3]);
+    if (i % 16384 === 0)
+      await progress(
+        25 + (5 * i) / snapshot.blocks.length,
+        `Indexing blocks ${i}/${snapshot.blocks.length}`,
+      );
   }
   const stateCache = new Map();
   async function stateModels(s) {
@@ -182,8 +201,15 @@ window.buildTerrain = async (snapshot) => {
     return variants;
   }
   // Preload each distinct model once before constructing batched geometry.
-  for (const s of distinctStates)
+  await progress(30, "Loading block models and textures", true);
+  let modelCount = 0;
+  for (const s of distinctStates) {
     await stateModels(s);
+    await progress(
+      30 + (5 * ++modelCount) / distinctStates.size,
+      `Loading block model ${modelCount}/${distinctStates.size}`,
+    );
+  }
   async function emit(block, m, v = {}) {
     const [bx, by, bz, s] = block;
     for (const e of m.elements || []) {
@@ -284,7 +310,14 @@ window.buildTerrain = async (snapshot) => {
       }
     }
   }
-  for (const b of snapshot.blocks) {
+  await progress(35, "Building terrain geometry", true);
+  for (let i = 0; i < snapshot.blocks.length; i++) {
+    const b = snapshot.blocks[i];
+    if (i % 256 === 0)
+      await progress(
+        35 + (20 * i) / snapshot.blocks.length,
+        `Building blocks ${i}/${snapshot.blocks.length}`,
+      );
     if (/minecraft:(water|lava)(\[|$)/.test(b[3])) {
       await emit(b, {
         textures: {
@@ -314,6 +347,8 @@ window.buildTerrain = async (snapshot) => {
     for (const { m, v } of await stateModels(b[3])) await emit(b, m, v);
   }
   occupancy.clear();
+  await progress(55, "Preparing the scene", true);
+  let batchCount = 0;
   for (const batch of batches.values()) {
     const g = new THREE.BufferGeometry();
     g.setAttribute(
@@ -340,6 +375,10 @@ window.buildTerrain = async (snapshot) => {
     batch.normals.length = 0;
     batch.uvs.length = 0;
     batch.indices.length = 0;
+    await progress(
+      55 + (5 * ++batchCount) / batches.size,
+      `Preparing geometry ${batchCount}/${batches.size}`,
+    );
   }
   window.terrainBounds = snapshot.bounds;
   return {

shared/processing-progress.ts

@@ -0,0 +1,21 @@
+export type ProcessingJob = {
+  state: string;
+  progress: number;
+  error: string | null;
+  updated_at: string;
+  lease_until: string | null;
+};
+
+// Stage ranges describe completed work, never an estimate of remaining time.
+export function processingStage(progress: number) {
+  if (progress < 20) return "Downloading and checking the export";
+  if (progress < 25) return "Reading the terrain snapshot";
+  if (progress < 30) return "Indexing terrain blocks";
+  if (progress < 35) return "Loading block models and textures";
+  if (progress < 55) return "Building terrain geometry";
+  if (progress < 60) return "Preparing the scene";
+  if (progress < 85) return "Rendering panorama faces";
+  if (progress < 90) return "Rendering the surface map";
+  if (progress < 100) return "Saving processed images";
+  return "Processing complete";
+}

worker/main.ts

@@ -32,6 +32,38 @@ async function processJob(job: {
   const abort = new AbortController(),
     directory = `.cache/jobs/${job.id}`;
   await mkdir(directory, { recursive: true });
+  let lastProgressWrite = 0;
+  let lastLoggedAt = 0;
+  let lastLoggedPercent = -1;
+  async function progress(percent: number, message: string) {
+    if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
+    const value = Math.max(0, Math.min(99, Math.floor(percent)));
+    const now = Date.now();
+    if (now - lastLoggedAt >= 30000 || value >= lastLoggedPercent + 5) {
+      console.log(`Job ${job.id}: ${value}% — ${message}`);
+      lastLoggedAt = now;
+      lastLoggedPercent = value;
+    }
+    if (now - lastProgressWrite < 2000 && value !== 99) return;
+    lastProgressWrite = now;
+    const rows = check(
+      await client
+        .from("upload_jobs")
+        .update({
+          progress: value,
+          error: null,
+          updated_at: new Date().toISOString(),
+        })
+        .eq("id", job.id)
+        .eq("lease_owner", worker)
+        .eq("state", "running")
+        .select("id"),
+    );
+    if (!rows.length) {
+      abort.abort();
+      throw new Error("Lease lost or cancelled");
+    }
+  }
   const heartbeat = setInterval(async () => {
     try {
       const rows = check(
@@ -67,6 +99,8 @@ async function processJob(job: {
         .single(),
     );
     const manifest = datasetSchema.parse(input.manifest);
+    await progress(0, `Downloading ${manifest.assets.length} export files`);
+    let downloaded = 0;
     for (const asset of manifest.assets) {
       if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
       const blob = check(
@@ -83,16 +117,15 @@ async function processJob(job: {
       const path = `${directory}/raw/${asset.path}`;
       await mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true });
       await writeFile(path, bytes);
+      await progress(
+        (20 * ++downloaded) / manifest.assets.length,
+        `Downloaded ${downloaded}/${manifest.assets.length} export files`,
+      );
     }
-    await client
-      .from("upload_jobs")
-      .update({ progress: 20 })
-      .eq("id", job.id)
-      .eq("lease_owner", worker)
-      .eq("state", "running");
     const output = `${directory}/media`;
     await mkdir(output, { recursive: true });
     if (manifest.source === "plugin") {
+      await progress(20, "Reading the terrain snapshot");
       const compressed = await readFile(`${directory}/raw/terrain.json.gz`);
       const terrain = JSON.parse(
         gunzipSync(compressed, {
@@ -123,6 +156,9 @@ async function processJob(job: {
         )
           throw new Error("Invalid block record");
       }
+      console.log(
+        `Job ${job.id}: ${terrain.blocks.length.toLocaleString()} blocks, ${manifest.viewpoints.length} panoramas; software rendering may take time.`,
+      );
       const report = await renderSnapshot(
         terrain,
         manifest.viewpoints,
@@ -130,15 +166,18 @@ async function processJob(job: {
         {
           size: Number(process.env.MINEGEO_FACE_SIZE || 1024),
           signal: abort.signal,
+          onProgress: ({ percent, message }) => progress(percent, message),
         },
       );
       console.log(`Rendered ${job.dataset_id}: ${JSON.stringify(report)}`);
     } else {
+      await progress(25, "Compressing the captured map");
       await sharp(await readFile(`${directory}/raw/map.png`), {
         limitInputPixels: 16777216,
       })
         .webp({ quality: 90 })
         .toFile(`${output}/map.webp`);
+      let processedFaces = 0;
       for (const point of manifest.viewpoints) {
         await mkdir(`${output}/${point.id}`, { recursive: true });
         for (const face of faces) {
@@ -163,13 +202,23 @@ async function processJob(job: {
               .resize(512, 512)
               .webp({ quality: 76 })
               .toFile(`${output}/${point.id}/preview.webp`);
+          await progress(
+            35 +
+              (55 * ++processedFaces) /
+                (manifest.viewpoints.length * faces.length),
+            `Compressing panorama faces ${processedFaces}/${manifest.viewpoints.length * faces.length}`,
+          );
         }
       }
     }
     const mapMeta = await sharp(`${output}/map.webp`).metadata();
     if (!mapMeta.width) throw new Error("Missing map output");
-    for (const entry of await readdir(output, { recursive: true })) {
+    await progress(90, "Saving processed images");
-      if (!entry.endsWith(".webp")) continue;
+    const images = (await readdir(output, { recursive: true })).filter(
+      (entry) => entry.endsWith(".webp"),
+    );
+    let uploaded = 0;
+    for (const entry of images) {
       if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
       const normalized = entry.replaceAll("\\", "/");
       check(
@@ -185,6 +234,10 @@ async function processJob(job: {
             },
           ),
       );
+      await progress(
+        90 + (9 * ++uploaded) / images.length,
+        `Saved ${uploaded}/${images.length} processed images`,
+      );
     }
     if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
     check(
@@ -195,6 +248,7 @@ async function processJob(job: {
         p_cover: manifest.viewpoints[0].id,
       }),
     );
+    console.log(`Job ${job.id}: 100% — private draft ready for preview.`);
   } catch (error) {
     const message =
       error instanceof Error ? error.message : "Processing failed";