Report renderer progress and detect stalled jobs

worker/main.ts

@@ -32,6 +32,38 @@ async function processJob(job: {
   const abort = new AbortController(),
     directory = `.cache/jobs/${job.id}`;
   await mkdir(directory, { recursive: true });
+  let lastProgressWrite = 0;
+  let lastLoggedAt = 0;
+  let lastLoggedPercent = -1;
+  async function progress(percent: number, message: string) {
+    if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
+    const value = Math.max(0, Math.min(99, Math.floor(percent)));
+    const now = Date.now();
+    if (now - lastLoggedAt >= 30000 || value >= lastLoggedPercent + 5) {
+      console.log(`Job ${job.id}: ${value}% — ${message}`);
+      lastLoggedAt = now;
+      lastLoggedPercent = value;
+    }
+    if (now - lastProgressWrite < 2000 && value !== 99) return;
+    lastProgressWrite = now;
+    const rows = check(
+      await client
+        .from("upload_jobs")
+        .update({
+          progress: value,
+          error: null,
+          updated_at: new Date().toISOString(),
+        })
+        .eq("id", job.id)
+        .eq("lease_owner", worker)
+        .eq("state", "running")
+        .select("id"),
+    );
+    if (!rows.length) {
+      abort.abort();
+      throw new Error("Lease lost or cancelled");
+    }
+  }
   const heartbeat = setInterval(async () => {
     try {
       const rows = check(
@@ -67,6 +99,8 @@ async function processJob(job: {
         .single(),
     );
     const manifest = datasetSchema.parse(input.manifest);
+    await progress(0, `Downloading ${manifest.assets.length} export files`);
+    let downloaded = 0;
     for (const asset of manifest.assets) {
       if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
       const blob = check(
@@ -83,16 +117,15 @@ async function processJob(job: {
       const path = `${directory}/raw/${asset.path}`;
       await mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true });
       await writeFile(path, bytes);
+      await progress(
+        (20 * ++downloaded) / manifest.assets.length,
+        `Downloaded ${downloaded}/${manifest.assets.length} export files`,
+      );
     }
-    await client
-      .from("upload_jobs")
-      .update({ progress: 20 })
-      .eq("id", job.id)
-      .eq("lease_owner", worker)
-      .eq("state", "running");
     const output = `${directory}/media`;
     await mkdir(output, { recursive: true });
     if (manifest.source === "plugin") {
+      await progress(20, "Reading the terrain snapshot");
       const compressed = await readFile(`${directory}/raw/terrain.json.gz`);
       const terrain = JSON.parse(
         gunzipSync(compressed, {
@@ -123,6 +156,9 @@ async function processJob(job: {
         )
           throw new Error("Invalid block record");
       }
+      console.log(
+        `Job ${job.id}: ${terrain.blocks.length.toLocaleString()} blocks, ${manifest.viewpoints.length} panoramas; software rendering may take time.`,
+      );
       const report = await renderSnapshot(
         terrain,
         manifest.viewpoints,
@@ -130,15 +166,18 @@ async function processJob(job: {
         {
           size: Number(process.env.MINEGEO_FACE_SIZE || 1024),
           signal: abort.signal,
+          onProgress: ({ percent, message }) => progress(percent, message),
         },
       );
       console.log(`Rendered ${job.dataset_id}: ${JSON.stringify(report)}`);
     } else {
+      await progress(25, "Compressing the captured map");
       await sharp(await readFile(`${directory}/raw/map.png`), {
         limitInputPixels: 16777216,
       })
         .webp({ quality: 90 })
         .toFile(`${output}/map.webp`);
+      let processedFaces = 0;
       for (const point of manifest.viewpoints) {
         await mkdir(`${output}/${point.id}`, { recursive: true });
         for (const face of faces) {
@@ -163,13 +202,23 @@ async function processJob(job: {
               .resize(512, 512)
               .webp({ quality: 76 })
               .toFile(`${output}/${point.id}/preview.webp`);
+          await progress(
+            35 +
+              (55 * ++processedFaces) /
+                (manifest.viewpoints.length * faces.length),
+            `Compressing panorama faces ${processedFaces}/${manifest.viewpoints.length * faces.length}`,
+          );
         }
       }
     }
     const mapMeta = await sharp(`${output}/map.webp`).metadata();
     if (!mapMeta.width) throw new Error("Missing map output");
-    for (const entry of await readdir(output, { recursive: true })) {
-      if (!entry.endsWith(".webp")) continue;
+    await progress(90, "Saving processed images");
+    const images = (await readdir(output, { recursive: true })).filter(
+      (entry) => entry.endsWith(".webp"),
+    );
+    let uploaded = 0;
+    for (const entry of images) {
       if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
       const normalized = entry.replaceAll("\\", "/");
       check(
@@ -185,6 +234,10 @@ async function processJob(job: {
             },
           ),
       );
+      await progress(
+        90 + (9 * ++uploaded) / images.length,
+        `Saved ${uploaded}/${images.length} processed images`,
+      );
     }
     if (abort.signal.aborted) throw new Error("Lease lost or cancelled");
     check(
@@ -195,6 +248,7 @@ async function processJob(job: {
         p_cover: manifest.viewpoints[0].id,
       }),
     );
+    console.log(`Job ${job.id}: 100% — private draft ready for preview.`);
   } catch (error) {
     const message =
       error instanceof Error ? error.message : "Processing failed";