Report renderer progress and detect stalled jobs

renderer/render.mjs

@@ -7,12 +7,12 @@ import { assetsRoot } from "../scripts/assets.mjs";
 /** @param {import('../shared/protocol').TerrainSnapshot} snapshot
  * @param {Array<{id:string,x:number,y:number,z:number}>} viewpoints
  * @param {string} output
- * @param {{size?:number,signal?:AbortSignal}} [options] */
+ * @param {{size?:number,signal?:AbortSignal,onProgress?:(event:{percent:number,message:string})=>Promise<void>}} [options] */
 export async function renderSnapshot(
   snapshot,
   viewpoints,
   output,
-  { size = 1024, signal } = {},
+  { size = 1024, signal, onProgress } = {},
 ) {
   const start = Date.now();
   await mkdir(output, { recursive: true });
@@ -26,14 +26,17 @@ export async function renderSnapshot(
         const send = async (chunk) => {
           if (res.destroyed) throw new Error("Terrain connection closed");
           await new Promise((resolve, reject) => {
-            res.write(chunk, (error) => error ? reject(error) : resolve());
+            res.write(chunk, (error) => (error ? reject(error) : resolve()));
           });
         };
         const { blocks, ...metadata } = snapshot;
         await send(JSON.stringify(metadata).slice(0, -1) + ',"blocks":[');
         for (let i = 0; i < blocks.length; i += 2048) {
           if (signal?.aborted) throw new Error("Rendering cancelled");
-          await send((i ? "," : "") + JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1));
+          await send(
+            (i ? "," : "") +
+              JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1),
+          );
         }
         res.end("]}");
         return;
@@ -63,19 +66,50 @@ export async function renderSnapshot(
       );
       res.end(await readFile(path));
     } catch {
-      if (res.headersSent) { res.destroy(); return; }
+      if (res.headersSent) {
+        res.destroy();
+        return;
+      }
       res.writeHead(404);
       res.end();
     }
   });
   await new Promise((r) => server.listen(0, "127.0.0.1", r));
   let browser;
+  let lastActivity = Date.now();
+  let failure;
+  const report = async (event) => {
+    lastActivity = Date.now();
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await onProgress?.(event);
+  };
+  const cancel = () => {
+    failure = new Error("Rendering cancelled or worker lease lost");
+    void browser?.close().catch(() => {});
+  };
+  const idleSeconds = Number(process.env.MINEGEO_RENDER_IDLE_SECONDS || 600);
+  if (!Number.isFinite(idleSeconds) || idleSeconds < 60) {
+    await new Promise((r) => server.close(r));
+    throw new Error("MINEGEO_RENDER_IDLE_SECONDS must be at least 60");
+  }
+  const watchdog = setInterval(() => {
+    if (Date.now() - lastActivity > idleSeconds * 1000) {
+      failure = new Error(
+        `Renderer stopped reporting progress for ${idleSeconds} seconds. Try a smaller export area or lower MINEGEO_FACE_SIZE.`,
+      );
+      void browser?.close().catch(() => {});
+    }
+  }, 10000);
+  signal?.addEventListener("abort", cancel, { once: true });
   try {
+    if (signal?.aborted) throw new Error("Rendering cancelled");
+    await report({ percent: 24, message: "Starting terrain renderer" });
     browser = await chromium.launch({
       headless: true,
       args: ["--use-angle=swiftshader", "--enable-unsafe-swiftshader"],
     });
     const page = await browser.newPage();
+    await page.exposeFunction("minegeoProgress", report);
     page.on("pageerror", (err) => console.error("Renderer:", err.message));
     await page.goto(`http://127.0.0.1:${server.address().port}`);
     await page.waitForFunction(() => window.rendererReady);
@@ -84,6 +118,11 @@ export async function renderSnapshot(
       if (!response.ok) throw new Error("Unable to load terrain snapshot");
       return window.buildTerrain(await response.json());
     });
+    await report({
+      percent: 60,
+      message: `Terrain ready: ${stats.blocks} blocks; rendering ${viewpoints.length} panoramas`,
+    });
+    let completedFaces = 0;
     for (const p of viewpoints) {
       if (signal?.aborted) throw new Error("Rendering cancelled");
       await mkdir(`${output}/${p.id}`, { recursive: true });
@@ -101,19 +140,32 @@ export async function renderSnapshot(
             .resize(512, 512)
             .webp({ quality: 76 })
             .toFile(`${output}/${p.id}/preview.webp`);
+        await report({
+          percent: 60 + (25 * ++completedFaces) / (viewpoints.length * 6),
+          message: `Panorama faces ${completedFaces}/${viewpoints.length * 6}`,
+        });
       }
     }
+    await report({ percent: 85, message: "Rendering the surface map" });
     const map = await page.evaluate(() => window.renderMap(1024));
     await sharp(Buffer.from(map.split(",")[1], "base64"))
       .webp({ quality: 90 })
       .toFile(`${output}/map.webp`);
+    await report({
+      percent: 90,
+      message: "Rendered all panoramas and the surface map",
+    });
     stats.renderMs = Date.now() - start;
     await writeFile(
       `${output}/render-report.json`,
       JSON.stringify(stats, null, 2),
     );
     return stats;
+  } catch (error) {
+    throw failure || error;
   } finally {
+    clearInterval(watchdog);
+    signal?.removeEventListener("abort", cancel);
     await browser?.close();
     await new Promise((r) => server.close(r));
   }