Report renderer progress and detect stalled jobs

renderer/render.mjs

@@ -7,12 +7,12 @@ import { assetsRoot } from "../scripts/assets.mjs";
 /** @param {import('../shared/protocol').TerrainSnapshot} snapshot
  * @param {Array<{id:string,x:number,y:number,z:number}>} viewpoints
  * @param {string} output
- * @param {{size?:number,signal?:AbortSignal}} [options] */
+ * @param {{size?:number,signal?:AbortSignal,onProgress?:(event:{percent:number,message:string})=>Promise<void>}} [options] */
 export async function renderSnapshot(
   snapshot,
   viewpoints,
   output,
-  { size = 1024, signal } = {},
+  { size = 1024, signal, onProgress } = {},
 ) {
   const start = Date.now();
   await mkdir(output, { recursive: true });
@@ -26,14 +26,17 @@ export async function renderSnapshot(
         const send = async (chunk) => {
           if (res.destroyed) throw new Error("Terrain connection closed");
           await new Promise((resolve, reject) => {
-            res.write(chunk, (error) => error ? reject(error) : resolve());
+            res.write(chunk, (error) => (error ? reject(error) : resolve()));
           });
         };
         const { blocks, ...metadata } = snapshot;
         await send(JSON.stringify(metadata).slice(0, -1) + ',"blocks":[');
         for (let i = 0; i < blocks.length; i += 2048) {
           if (signal?.aborted) throw new Error("Rendering cancelled");
-          await send((i ? "," : "") + JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1));
+          await send(
+            (i ? "," : "") +
+              JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1),
+          );
         }
         res.end("]}");
         return;
@@ -63,19 +66,50 @@ export async function renderSnapshot(
       );
       res.end(await readFile(path));
     } catch {
-      if (res.headersSent) { res.destroy(); return; }
+      if (res.headersSent) {
+        res.destroy();
+        return;
+      }
       res.writeHead(404);
       res.end();
     }
   });
   await new Promise((r) => server.listen(0, "127.0.0.1", r));
   let browser;
+  let lastActivity = Date.now();
+  let failure;
+  const report = async (event) => {
+    lastActivity = Date.now();
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await onProgress?.(event);
+  };
+  const cancel = () => {
+    failure = new Error("Rendering cancelled or worker lease lost");
+    void browser?.close().catch(() => {});
+  };
+  const idleSeconds = Number(process.env.MINEGEO_RENDER_IDLE_SECONDS || 600);
+  if (!Number.isFinite(idleSeconds) || idleSeconds < 60) {
+    await new Promise((r) => server.close(r));
+    throw new Error("MINEGEO_RENDER_IDLE_SECONDS must be at least 60");
+  }
+  const watchdog = setInterval(() => {
+    if (Date.now() - lastActivity > idleSeconds * 1000) {
+      failure = new Error(
+        `Renderer stopped reporting progress for ${idleSeconds} seconds. Try a smaller export area or lower MINEGEO_FACE_SIZE.`,
+      );
+      void browser?.close().catch(() => {});
+    }
+  }, 10000);
+  signal?.addEventListener("abort", cancel, { once: true });
   try {
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await report({ percent: 24, message: "Starting terrain renderer" });
     browser = await chromium.launch({
       headless: true,
       args: ["--use-angle=swiftshader", "--enable-unsafe-swiftshader"],
     });
     const page = await browser.newPage();
+    await page.exposeFunction("minegeoProgress", report);
     page.on("pageerror", (err) => console.error("Renderer:", err.message));
     await page.goto(`http://127.0.0.1:${server.address().port}`);
     await page.waitForFunction(() => window.rendererReady);
@@ -84,6 +118,11 @@ export async function renderSnapshot(
       if (!response.ok) throw new Error("Unable to load terrain snapshot");
       return window.buildTerrain(await response.json());
     });
+    await report({
+      percent: 60,
+      message: `Terrain ready: ${stats.blocks} blocks; rendering ${viewpoints.length} panoramas`,
+    });
+    let completedFaces = 0;
     for (const p of viewpoints) {
       if (signal?.aborted) throw new Error("Rendering cancelled");
       await mkdir(`${output}/${p.id}`, { recursive: true });
@@ -101,19 +140,32 @@ export async function renderSnapshot(
             .resize(512, 512)
             .webp({ quality: 76 })
             .toFile(`${output}/${p.id}/preview.webp`);
+        await report({
+          percent: 60 + (25 * ++completedFaces) / (viewpoints.length * 6),
+          message: `Panorama faces ${completedFaces}/${viewpoints.length * 6}`,
+        });
       }
     }
+    await report({ percent: 85, message: "Rendering the surface map" });
     const map = await page.evaluate(() => window.renderMap(1024));
     await sharp(Buffer.from(map.split(",")[1], "base64"))
       .webp({ quality: 90 })
       .toFile(`${output}/map.webp`);
+    await report({
+      percent: 90,
+      message: "Rendered all panoramas and the surface map",
+    });
     stats.renderMs = Date.now() - start;
     await writeFile(
       `${output}/render-report.json`,
       JSON.stringify(stats, null, 2),
     );
     return stats;
+  } catch (error) {
+    throw failure || error;
   } finally {
+    clearInterval(watchdog);
+    signal?.removeEventListener("abort", cancel);
     await browser?.close();
     await new Promise((r) => server.close(r));
   }

renderer/scene.js

@@ -162,9 +162,28 @@ window.buildTerrain = async (snapshot) => {
     unsupported = new Set();
   const occupancy = new Map();
   const distinctStates = new Set();
-  for (const b of snapshot.blocks) {
+  let lastProgress = 0;
+  async function progress(percent, message, force = false) {
+    if (!force && performance.now() - lastProgress < 3000) return;
+    lastProgress = performance.now();
+    // Yield to browser tasks so bindings, cancellation and timers can run.
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    await window.minegeoProgress?.({ percent, message });
+  }
+  await progress(
+    25,
+    `Indexing ${snapshot.blocks.length.toLocaleString()} terrain blocks`,
+    true,
+  );
+  for (let i = 0; i < snapshot.blocks.length; i++) {
+    const b = snapshot.blocks[i];
     occupancy.set(`${b[0]},${b[1]},${b[2]}`, b[3]);
     distinctStates.add(b[3]);
+    if (i % 16384 === 0)
+      await progress(
+        25 + (5 * i) / snapshot.blocks.length,
+        `Indexing blocks ${i}/${snapshot.blocks.length}`,
+      );
   }
   const stateCache = new Map();
   async function stateModels(s) {
@@ -182,8 +201,15 @@ window.buildTerrain = async (snapshot) => {
     return variants;
   }
   // Preload each distinct model once before constructing batched geometry.
-  for (const s of distinctStates)
+  await progress(30, "Loading block models and textures", true);
+  let modelCount = 0;
+  for (const s of distinctStates) {
     await stateModels(s);
+    await progress(
+      30 + (5 * ++modelCount) / distinctStates.size,
+      `Loading block model ${modelCount}/${distinctStates.size}`,
+    );
+  }
   async function emit(block, m, v = {}) {
     const [bx, by, bz, s] = block;
     for (const e of m.elements || []) {
@@ -284,7 +310,14 @@ window.buildTerrain = async (snapshot) => {
       }
     }
   }
-  for (const b of snapshot.blocks) {
+  await progress(35, "Building terrain geometry", true);
+  for (let i = 0; i < snapshot.blocks.length; i++) {
+    const b = snapshot.blocks[i];
+    if (i % 256 === 0)
+      await progress(
+        35 + (20 * i) / snapshot.blocks.length,
+        `Building blocks ${i}/${snapshot.blocks.length}`,
+      );
     if (/minecraft:(water|lava)(\[|$)/.test(b[3])) {
       await emit(b, {
         textures: {
@@ -314,6 +347,8 @@ window.buildTerrain = async (snapshot) => {
     for (const { m, v } of await stateModels(b[3])) await emit(b, m, v);
   }
   occupancy.clear();
+  await progress(55, "Preparing the scene", true);
+  let batchCount = 0;
   for (const batch of batches.values()) {
     const g = new THREE.BufferGeometry();
     g.setAttribute(
@@ -340,6 +375,10 @@ window.buildTerrain = async (snapshot) => {
     batch.normals.length = 0;
     batch.uvs.length = 0;
     batch.indices.length = 0;
+    await progress(
+      55 + (5 * ++batchCount) / batches.size,
+      `Preparing geometry ${batchCount}/${batches.size}`,
+    );
   }
   window.terrainBounds = snapshot.bounds;
   return {