Bound renderer memory use and configure worker heap ceiling
This commit is contained in:
@@ -13,4 +13,6 @@ SUPABASE_SECRET_KEY=
|
|||||||
# Optional direct PostgreSQL connection for maintenance. Runtime uses the Supabase API.
|
# Optional direct PostgreSQL connection for maintenance. Runtime uses the Supabase API.
|
||||||
DATABASE_URL=
|
DATABASE_URL=
|
||||||
MINEGEO_FACE_SIZE=1024
|
MINEGEO_FACE_SIZE=1024
|
||||||
|
# Worker heap ceiling in MiB; does not reserve memory or change the web server heap.
|
||||||
|
MINEGEO_WORKER_HEAP_MB=4096
|
||||||
MINEGEO_MAX_UPLOAD_BYTES=268435456
|
MINEGEO_MAX_UPLOAD_BYTES=268435456
|
||||||
|
|||||||
@@ -19,6 +19,25 @@ export async function renderSnapshot(
|
|||||||
const server = createServer(async (req, res) => {
|
const server = createServer(async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const url = new URL(req.url, "http://localhost");
|
const url = new URL(req.url, "http://localhost");
|
||||||
|
if (url.pathname === "/terrain.json") {
|
||||||
|
res.setHeader("Content-Type", "application/json");
|
||||||
|
// Avoid Playwright's recursive protocol serialization of millions of
|
||||||
|
// block records. Keep serialization buffers bounded and honor backpressure.
|
||||||
|
const send = async (chunk) => {
|
||||||
|
if (res.destroyed) throw new Error("Terrain connection closed");
|
||||||
|
await new Promise((resolve, reject) => {
|
||||||
|
res.write(chunk, (error) => error ? reject(error) : resolve());
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const { blocks, ...metadata } = snapshot;
|
||||||
|
await send(JSON.stringify(metadata).slice(0, -1) + ',"blocks":[');
|
||||||
|
for (let i = 0; i < blocks.length; i += 2048) {
|
||||||
|
if (signal?.aborted) throw new Error("Rendering cancelled");
|
||||||
|
await send((i ? "," : "") + JSON.stringify(blocks.slice(i, i + 2048)).slice(1, -1));
|
||||||
|
}
|
||||||
|
res.end("]}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
let path =
|
let path =
|
||||||
url.pathname === "/three.js"
|
url.pathname === "/three.js"
|
||||||
? "node_modules/three/build/three.module.js"
|
? "node_modules/three/build/three.module.js"
|
||||||
@@ -44,6 +63,7 @@ export async function renderSnapshot(
|
|||||||
);
|
);
|
||||||
res.end(await readFile(path));
|
res.end(await readFile(path));
|
||||||
} catch {
|
} catch {
|
||||||
|
if (res.headersSent) { res.destroy(); return; }
|
||||||
res.writeHead(404);
|
res.writeHead(404);
|
||||||
res.end();
|
res.end();
|
||||||
}
|
}
|
||||||
@@ -59,7 +79,11 @@ export async function renderSnapshot(
|
|||||||
page.on("pageerror", (err) => console.error("Renderer:", err.message));
|
page.on("pageerror", (err) => console.error("Renderer:", err.message));
|
||||||
await page.goto(`http://127.0.0.1:${server.address().port}`);
|
await page.goto(`http://127.0.0.1:${server.address().port}`);
|
||||||
await page.waitForFunction(() => window.rendererReady);
|
await page.waitForFunction(() => window.rendererReady);
|
||||||
const stats = await page.evaluate((s) => window.buildTerrain(s), snapshot);
|
const stats = await page.evaluate(async () => {
|
||||||
|
const response = await fetch("/terrain.json");
|
||||||
|
if (!response.ok) throw new Error("Unable to load terrain snapshot");
|
||||||
|
return window.buildTerrain(await response.json());
|
||||||
|
});
|
||||||
for (const p of viewpoints) {
|
for (const p of viewpoints) {
|
||||||
if (signal?.aborted) throw new Error("Rendering cancelled");
|
if (signal?.aborted) throw new Error("Rendering cancelled");
|
||||||
await mkdir(`${output}/${p.id}`, { recursive: true });
|
await mkdir(`${output}/${p.id}`, { recursive: true });
|
||||||
|
|||||||
@@ -160,9 +160,12 @@ window.buildTerrain = async (snapshot) => {
|
|||||||
const started = performance.now(),
|
const started = performance.now(),
|
||||||
batches = new Map(),
|
batches = new Map(),
|
||||||
unsupported = new Set();
|
unsupported = new Set();
|
||||||
const occupancy = new Map(
|
const occupancy = new Map();
|
||||||
snapshot.blocks.map((b) => [`${b[0]},${b[1]},${b[2]}`, b[3]]),
|
const distinctStates = new Set();
|
||||||
);
|
for (const b of snapshot.blocks) {
|
||||||
|
occupancy.set(`${b[0]},${b[1]},${b[2]}`, b[3]);
|
||||||
|
distinctStates.add(b[3]);
|
||||||
|
}
|
||||||
const stateCache = new Map();
|
const stateCache = new Map();
|
||||||
async function stateModels(s) {
|
async function stateModels(s) {
|
||||||
if (stateCache.has(s)) return stateCache.get(s);
|
if (stateCache.has(s)) return stateCache.get(s);
|
||||||
@@ -179,7 +182,7 @@ window.buildTerrain = async (snapshot) => {
|
|||||||
return variants;
|
return variants;
|
||||||
}
|
}
|
||||||
// Preload each distinct model once before constructing batched geometry.
|
// Preload each distinct model once before constructing batched geometry.
|
||||||
for (const s of new Set(snapshot.blocks.map((b) => b[3])))
|
for (const s of distinctStates)
|
||||||
await stateModels(s);
|
await stateModels(s);
|
||||||
async function emit(block, m, v = {}) {
|
async function emit(block, m, v = {}) {
|
||||||
const [bx, by, bz, s] = block;
|
const [bx, by, bz, s] = block;
|
||||||
@@ -310,6 +313,7 @@ window.buildTerrain = async (snapshot) => {
|
|||||||
}
|
}
|
||||||
for (const { m, v } of await stateModels(b[3])) await emit(b, m, v);
|
for (const { m, v } of await stateModels(b[3])) await emit(b, m, v);
|
||||||
}
|
}
|
||||||
|
occupancy.clear();
|
||||||
for (const batch of batches.values()) {
|
for (const batch of batches.values()) {
|
||||||
const g = new THREE.BufferGeometry();
|
const g = new THREE.BufferGeometry();
|
||||||
g.setAttribute(
|
g.setAttribute(
|
||||||
@@ -331,6 +335,11 @@ window.buildTerrain = async (snapshot) => {
|
|||||||
opacity: batch.tex.image.src.includes("water") ? 0.8 : 1,
|
opacity: batch.tex.image.src.includes("water") ? 0.8 : 1,
|
||||||
});
|
});
|
||||||
scene.add(new THREE.Mesh(g, mat));
|
scene.add(new THREE.Mesh(g, mat));
|
||||||
|
// Drop the JS number arrays as each GPU geometry is completed.
|
||||||
|
batch.positions.length = 0;
|
||||||
|
batch.normals.length = 0;
|
||||||
|
batch.uvs.length = 0;
|
||||||
|
batch.indices.length = 0;
|
||||||
}
|
}
|
||||||
window.terrainBounds = snapshot.bounds;
|
window.terrainBounds = snapshot.bounds;
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -34,4 +34,10 @@ run([
|
|||||||
"--hostname",
|
"--hostname",
|
||||||
process.env.MINEGEO_HOST || "127.0.0.1",
|
process.env.MINEGEO_HOST || "127.0.0.1",
|
||||||
]);
|
]);
|
||||||
run(["--import", "tsx", "worker/main.ts"]);
|
const workerHeap = Number(process.env.MINEGEO_WORKER_HEAP_MB || 4096);
|
||||||
|
if (!Number.isInteger(workerHeap) || workerHeap < 512 || workerHeap > 32768) {
|
||||||
|
console.error("MINEGEO_WORKER_HEAP_MB must be an integer between 512 and 32768.");
|
||||||
|
stop(1);
|
||||||
|
} else {
|
||||||
|
run([`--max-old-space-size=${workerHeap}`, "--import", "tsx", "worker/main.ts"]);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user