Fail fast on missing terrain assets and recover empty renders

scripts/assets.mjs

@@ -1,36 +1,90 @@
-import { mkdir, readFile, writeFile, access } from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { createHash } from "node:crypto";
 import { unzipSync } from "fflate";
 export const assetsRoot = ".cache/minecraft/1.21.11";
-export async function prepareAssets() {
+const completionMarker = `${assetsRoot}/.minegeo-assets.json`;
+const requiredAssets = [
+  "blockstates/grass_block.json",
+  "blockstates/stone.json",
+  "models/block/block.json",
+  "models/block/cube.json",
+  "models/block/cube_all.json",
+  "textures/block/grass_block_top.png",
+  "textures/block/dirt.png",
+  "textures/block/stone.png",
+  "textures/block/water_still.png",
+];
+async function assetsReady() {
   try {
-    await access(`${assetsRoot}/assets/minecraft/blockstates/grass_block.json`);
-    return;
-  } catch {}
+    const marker = JSON.parse(await readFile(completionMarker, "utf8"));
+    if (
+      marker.format !== "minegeo-assets/v1" ||
+      marker.minecraftVersion !== "1.21.11" ||
+      !marker.files
+    )
+      return false;
+    // A blockstate alone is insufficient: interrupted setup can leave models
+    // and textures absent. The marker is only written after all extraction.
+    for (const asset of requiredAssets) {
+      const bytes = await readFile(`${assetsRoot}/assets/minecraft/${asset}`);
+      if (asset.endsWith(".json")) JSON.parse(bytes.toString());
+      else if (
+        bytes.length < 8 ||
+        bytes.subarray(0, 8).toString("hex") !== "89504e470d0a1a0a"
+      )
+        return false;
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
+export async function prepareAssets() {
+  if (await assetsReady()) return;
+  console.log(
+    "Preparing Minecraft 1.21.11 models and textures for terrain rendering…",
+  );
   const manifest = await (
     await fetch(
       "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json",
+      { signal: AbortSignal.timeout(60000) },
     )
   ).json();
   const entry = manifest.versions.find((v) => v.id === "1.21.11");
   if (!entry) throw new Error("Official Minecraft 1.21.11 metadata missing");
-  const version = await (await fetch(entry.url)).json();
+  const version = await (
+    await fetch(entry.url, { signal: AbortSignal.timeout(60000) })
+  ).json();
   await mkdir(".cache/minecraft", { recursive: true });
   const jarPath = ".cache/minecraft/client-1.21.11.jar";
   let jar;
   try {
     jar = await readFile(jarPath);
   } catch {
-    const response = await fetch(version.downloads.client.url);
-    if (!response.ok) throw new Error("Minecraft asset download failed");
-    jar = Buffer.from(await response.arrayBuffer());
-    await writeFile(jarPath, jar);
+    /* No cached client yet. */
   }
   if (
+    !jar ||
     createHash("sha1").update(jar).digest("hex") !==
-    version.downloads.client.sha1
-  )
-    throw new Error("Minecraft client checksum mismatch");
+      version.downloads.client.sha1
+  ) {
+    const response = await fetch(version.downloads.client.url, {
+      signal: AbortSignal.timeout(120000),
+    });
+    if (!response.ok)
+      throw new Error(
+        `Minecraft asset download failed: HTTP ${response.status}`,
+      );
+    jar = Buffer.from(await response.arrayBuffer());
+    if (
+      createHash("sha1").update(jar).digest("hex") !==
+      version.downloads.client.sha1
+    )
+      throw new Error(
+        "Minecraft client checksum mismatch; downloaded assets were not saved",
+      );
+    await writeFile(jarPath, jar);
+  }
   const entries = unzipSync(jar, {
     filter: (f) =>
       /^assets\/minecraft\/(models|blockstates|textures\/block)\//.test(
@@ -42,6 +96,20 @@ export async function prepareAssets() {
     await mkdir(path.slice(0, path.lastIndexOf("/")), { recursive: true });
     await writeFile(path, bytes);
   }
+  // Completion is recorded last, so a killed extraction is repaired next time.
+  await writeFile(
+    completionMarker,
+    JSON.stringify({
+      format: "minegeo-assets/v1",
+      minecraftVersion: "1.21.11",
+      clientSha1: version.downloads.client.sha1,
+      files: Object.keys(entries).length,
+    }),
+  );
+  if (!(await assetsReady()))
+    throw new Error(
+      "Minecraft rendering assets are incomplete or corrupt. Run npm run setup to repair them.",
+    );
   console.log(
     `Prepared ${Object.keys(entries).length} verified vanilla model/texture assets.`,
   );

scripts/reprocess.mjs

@@ -0,0 +1,138 @@
+import { existsSync } from "node:fs";
+import { createClient } from "@supabase/supabase-js";
+import { prepareAssets } from "./assets.mjs";
+
+if (existsSync(".env.local")) process.loadEnvFile(".env.local");
+const id = process.argv[2];
+if (
+  !id ||
+  !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)
+)
+  throw new Error(
+    "Usage: npm run reprocess -- <dataset UUID>. Stop the MineGeo worker first.",
+  );
+if (!process.env.NEXT_PUBLIC_SUPABASE_URL || !process.env.SUPABASE_SECRET_KEY)
+  throw new Error("Configure Supabase in .env.local before reprocessing.");
+const client = createClient(
+  process.env.NEXT_PUBLIC_SUPABASE_URL,
+  process.env.SUPABASE_SECRET_KEY,
+  {
+    auth: { persistSession: false, autoRefreshToken: false },
+  },
+);
+function check(result) {
+  if (result.error) throw new Error(result.error.message);
+  return result.data;
+}
+const dataset = check(
+  await client.from("datasets").select("*").eq("id", id).single(),
+);
+if (
+  dataset.revision_id ||
+  !["draft", "review", "rejected", "failed"].includes(dataset.state)
+)
+  throw new Error(
+    "Only an unpublished, inactive draft can be reprocessed. Published revisions cannot be changed.",
+  );
+const job = check(
+  await client.from("upload_jobs").select("*").eq("dataset_id", id).single(),
+);
+if (!["complete", "failed", "cancelled"].includes(job.state))
+  throw new Error(
+    "This job is already queued or running. Stop the worker and wait for it to finish before retrying.",
+  );
+const input = check(
+  await client
+    .from("dataset_inputs")
+    .select("validated,manifest")
+    .eq("dataset_id", id)
+    .single(),
+);
+if (dataset.source === "plugin") await prepareAssets();
+
+// This maintenance command runs with the worker stopped. Compare-and-set
+// guards reject intervening publication, ownership of a job, or another repair.
+const changed = check(
+  await client
+    .from("datasets")
+    .update({
+      state: "processing",
+      rejection_reason: null,
+      map_path: null,
+      cover_path: null,
+      panorama_count: 0,
+      updated_at: new Date().toISOString(),
+    })
+    .eq("id", id)
+    .eq("state", dataset.state)
+    .is("revision_id", null)
+    .select("id"),
+);
+if (changed.length !== 1)
+  throw new Error(
+    "Dataset state changed; nothing was queued. Run the command again.",
+  );
+try {
+  check(
+    await client
+      .from("dataset_inputs")
+      .update({ validated: false })
+      .eq("dataset_id", id),
+  );
+  const queued = check(
+    await client
+      .from("upload_jobs")
+      .update({
+        state: "queued",
+        attempts: 0,
+        progress: 0,
+        error: null,
+        lease_owner: null,
+        lease_until: null,
+        updated_at: new Date().toISOString(),
+      })
+      .eq("id", job.id)
+      .eq("state", job.state)
+      .eq("updated_at", job.updated_at)
+      .select("id"),
+  );
+  if (queued.length !== 1)
+    throw new Error(
+      "Job state changed while queuing; stop the worker before retrying.",
+    );
+} catch (error) {
+  // A transport error can occur after an update committed. If the job is
+  // queued/running, leave its dataset processing; otherwise restore its draft.
+  const current = await client
+    .from("upload_jobs")
+    .select("state")
+    .eq("id", job.id)
+    .single();
+  if (!current.error && !["queued", "running"].includes(current.data.state)) {
+    check(
+      await client
+        .from("datasets")
+        .update({
+          state: dataset.state,
+          rejection_reason: dataset.rejection_reason,
+          map_path: dataset.map_path,
+          cover_path: dataset.cover_path,
+          panorama_count: dataset.panorama_count,
+          updated_at: new Date().toISOString(),
+        })
+        .eq("id", id)
+        .eq("state", "processing")
+        .is("revision_id", null),
+    );
+    check(
+      await client
+        .from("dataset_inputs")
+        .update({ validated: input.validated })
+        .eq("dataset_id", id),
+    );
+  }
+  throw error;
+}
+console.log(
+  `Queued dataset ${id} for a fresh render. Start MineGeo; no new export or upload is needed.`,
+);

scripts/setup.mjs

@@ -7,6 +7,8 @@ await mkdir(".data", { recursive: true });
 await mkdir("public/generated", { recursive: true });
 await mkdir("public/tools/mod", { recursive: true });
 await mkdir("public/tools/plugin", { recursive: true });
+// Prepare terrain assets even if browser installation subsequently fails.
+await prepareAssets();
 await new Promise((resolve, reject) => {
   const p = spawn(
     process.execPath,
@@ -19,7 +21,6 @@ await new Promise((resolve, reject) => {
       : reject(new Error(`Chromium installation exited ${code}`)),
   );
 });
-await prepareAssets();
 process.loadEnvFile(".env.local");
 const configured = Boolean(
   process.env.NEXT_PUBLIC_SUPABASE_URL &&

scripts/test-renderer.mjs

@@ -0,0 +1,140 @@
+import assert from "node:assert/strict";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { gunzipSync } from "node:zlib";
+import sharp from "sharp";
+import { renderSnapshot } from "../renderer/render.mjs";
+import { assertTerrainImage } from "../renderer/image-validation.mjs";
+
+// Exercise the product's rendering worker without launching a Minecraft server
+// or driving the website. A cached real export can also be supplied as argv[2].
+const output = ".cache/renderer-regression";
+const snapshot = {
+  bounds: { minX: -24, minZ: -24, maxX: 24, maxZ: 24 },
+  coverage: [],
+  blocks: [],
+};
+for (let x = -24; x < 24; x++) {
+  for (let z = -24; z < 24; z++) {
+    snapshot.blocks.push([x, 63, z, "minecraft:dirt"]);
+    snapshot.blocks.push([
+      x,
+      64,
+      z,
+      x > 12 && z < -12
+        ? "minecraft:water[level=0]"
+        : "minecraft:grass_block[snowy=false]",
+    ]);
+  }
+}
+for (let x = -4; x <= 4; x++) {
+  for (let z = -4; z <= 4; z++) {
+    for (let y = 65; y <= 71; y++) {
+      if (Math.abs(x) === 4 || Math.abs(z) === 4 || y === 71) {
+        snapshot.blocks.push([
+          x,
+          y,
+          z,
+          y === 71 ? "minecraft:oak_planks" : "minecraft:stone_bricks",
+        ]);
+      }
+    }
+  }
+}
+for (let y = 65; y <= 70; y++)
+  snapshot.blocks.push([-16, y, 0, "minecraft:oak_log[axis=y]"]);
+for (let x = -18; x <= -14; x++) {
+  for (let z = -2; z <= 2; z++) {
+    snapshot.blocks.push([
+      x,
+      71,
+      z,
+      "minecraft:oak_leaves[distance=1,persistent=true,waterlogged=false]",
+    ]);
+  }
+}
+snapshot.blocks.push([
+  5,
+  65,
+  0,
+  "minecraft:oak_stairs[facing=east,half=bottom,shape=straight,waterlogged=false]",
+]);
+let terrain = snapshot;
+let points = [
+  [-12.5, 12.5],
+  [12.5, 12.5],
+  [12.5, -12.5],
+  [-12.5, -12.5],
+  [0.5, 18.5],
+].map(([x, z], i) => ({ id: `p${i}`, x, y: 66.62, z }));
+let source = "grass, dirt, water, tree and building regression fixture";
+if (process.argv[2]) {
+  const directory = process.argv[2];
+  const manifest = JSON.parse(
+    await readFile(`${directory}/manifest.json`, "utf8"),
+  );
+  terrain = JSON.parse(
+    gunzipSync(await readFile(`${directory}/terrain.json.gz`)).toString(),
+  );
+  points = manifest.viewpoints;
+  source = "cached Paper export (no server started)";
+}
+await mkdir(output, { recursive: true });
+const report = await renderSnapshot(terrain, points, output, { size: 256 });
+assert.ok(report.meshes > 0, "terrain must produce meshes");
+assert.ok(report.emittedFaces > 0, "terrain must produce faces");
+assert.ok(report.hiddenFaces > 0, "interior terrain faces must be culled");
+assert.ok(report.geometryBytes > 0, "geometry must reach the renderer");
+assert.equal(
+  report.unsupported.some((s) =>
+    /minecraft:(stone|dirt|grass_block|water)(\[|$)/.test(s),
+  ),
+  false,
+);
+await assertTerrainImage(await readFile(`${output}/map.webp`), "Surface map");
+for (const point of points) {
+  for (const face of ["px", "nx", "py", "ny", "pz", "nz"]) {
+    const metadata = await sharp(
+      await readFile(`${output}/${point.id}/${face}.webp`),
+    ).metadata();
+    assert.equal(metadata.format, "webp");
+    assert.equal(metadata.width, 256);
+    assert.equal(metadata.height, 256);
+  }
+  await assertTerrainImage(
+    await readFile(`${output}/${point.id}/ny.webp`),
+    `Panorama ${point.id}, downward face`,
+  );
+}
+
+// The earlier renderer silently accepted this case with zero meshes and faces.
+const missingModel = {
+  ...snapshot,
+  blocks: [[0, 64, 0, "minecraft:minegeo_missing_model"]],
+};
+await assert.rejects(
+  renderSnapshot(missingModel, [points[0]], `${output}/missing-model`, {
+    size: 256,
+  }),
+  /No terrain models could be rendered/,
+);
+await writeFile(
+  `${output}/verification.json`,
+  JSON.stringify(
+    {
+      source,
+      ...report,
+      checks: [
+        "nonempty terrain",
+        "valid WebP cubemap faces",
+        "surface visible below every viewpoint",
+        "missing model rejected",
+      ],
+    },
+    null,
+    2,
+  ),
+);
+console.log(
+  "Renderer regression passed:",
+  JSON.stringify({ source, ...report }),
+);