Add detailed diagnostics to Dokploy deployment workflow

.gitea/workflows/deploy-dokploy.yml

@@ -33,8 +33,11 @@ jobs:
           archive_file="$(mktemp "$temp_root/minegeo-dokploy-source.XXXXXX")"
           response_file="$(mktemp "$temp_root/minegeo-dokploy-response.XXXXXX")"
           deployment_file="$(mktemp "$temp_root/minegeo-dokploy-status.XXXXXX")"
+          metrics_file="$(mktemp "$temp_root/minegeo-dokploy-metrics.XXXXXX")"
+          error_file="$(mktemp "$temp_root/minegeo-dokploy-errors.XXXXXX")"
+          headers_file="$(mktemp "$temp_root/minegeo-dokploy-headers.XXXXXX")"
           cleanup() {
-            rm -f "$auth_file" "$archive_file" "$response_file" "$deployment_file"
+            rm -f "$auth_file" "$archive_file" "$response_file" "$deployment_file" "$metrics_file" "$error_file" "$headers_file"
           }
           trap cleanup EXIT
 
@@ -42,24 +45,57 @@ jobs:
           printf 'x-api-key: %s\n' "$DOKPLOY_API_KEY" > "$auth_file"
           unset DOKPLOY_API_KEY
 
+          echo "[Dokploy] Workflow revision: $(git rev-parse HEAD)"
+          curl --version | head -n 2
+          node scripts/dokploy-diagnostics.mjs environment "$DOKPLOY_URL"
+
+          dokploy_request() {
+            label="$1"
+            body_file="$2"
+            shift 2
+            : > "$body_file"
+            : > "$headers_file"
+            : > "$metrics_file"
+            : > "$error_file"
+            echo "[Dokploy] Starting $label at $(date -u '+%Y-%m-%dT%H:%M:%SZ')" >&2
+            if curl --silent --show-error \
+              --header "@$auth_file" \
+              --dump-header "$headers_file" \
+              --output "$body_file" \
+              --write-out '%{http_code}\nremote_ip=%{remote_ip} remote_port=%{remote_port} local_ip=%{local_ip}\ndns=%{time_namelookup}s connect=%{time_connect}s tls=%{time_appconnect}s first_byte=%{time_starttransfer}s total=%{time_total}s\nssl_verify_result=%{ssl_verify_result} redirects=%{num_redirects} uploaded=%{size_upload}B downloaded=%{size_download}B\n' \
+              "$@" > "$metrics_file" 2> "$error_file"; then
+              curl_exit=0
+            else
+              curl_exit=$?
+            fi
+            node scripts/dokploy-diagnostics.mjs request "$label" "$curl_exit" \
+              "$metrics_file" "$error_file" "$headers_file" "$body_file" "$auth_file" >&2
+            http_code="$(head -n 1 "$metrics_file")"
+            if [ "$curl_exit" -ne 0 ]; then
+              echo "[Dokploy] $label failed: curl exit $curl_exit, HTTP ${http_code:-000}." >&2
+              return 1
+            fi
+            case "$http_code" in
+              2??) ;;
+              *) echo "[Dokploy] $label failed: HTTP ${http_code:-000}." >&2; return 1 ;;
+            esac
+          }
+
           # Only committed files from this revision are uploaded. This excludes
           # .git metadata, local .env files, caches, and untracked workspace data.
           git archive --format=zip --output="$archive_file" HEAD
+          echo "[Dokploy] Source archive size: $(wc -c < "$archive_file") bytes"
 
-          before_code="$(curl --silent --get \
+          dokploy_request "API preflight GET /api/deployment.all" "$deployment_file" --get \
-            --header "@$auth_file" \
             --data-urlencode "applicationId=$DOKPLOY_APPLICATION_ID" \
             --connect-timeout 10 --max-time 30 \
-            --output "$deployment_file" --write-out '%{http_code}' \
+            "$DOKPLOY_URL/api/deployment.all"
-            "$DOKPLOY_URL/api/deployment.all" 2>/dev/null || true)"
-          case "$before_code" in
-            2??) ;;
-            *) echo "Dokploy API preflight failed (HTTP ${before_code:-000}); response withheld." >&2; exit 1 ;;
-          esac
 
           previous_id="$(DOKPLOY_STATUS_FILE="$deployment_file" node <<'NODE'
           const fs = require("node:fs");
-          const payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8"));
+          let payload;
+          try { payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); }
+          catch { console.error("Dokploy deployment list was not valid JSON."); process.exit(2); }
           const rows = Array.isArray(payload)
             ? payload
             : [payload?.data, payload?.deployments, payload?.items].find(Array.isArray);
@@ -68,38 +104,31 @@ jobs:
           process.stdout.write(String(latest?.id ?? latest?.deploymentId ?? ""));
           NODE
           )" || {
+            node scripts/dokploy-diagnostics.mjs request "API preflight response parsing" parse-error \
+              "$metrics_file" "$error_file" "$headers_file" "$deployment_file" "$auth_file" >&2
             echo "Dokploy returned an unexpected deployment list; source was not uploaded." >&2
             exit 1
           }
 
-          upload_code="$(curl --silent \
+          dokploy_request "Source upload POST /api/application.dropDeployment" "$response_file" \
-            --header "@$auth_file" \
             --form-string "applicationId=$DOKPLOY_APPLICATION_ID" \
             --form "zip=@$archive_file;type=application/zip" \
             --connect-timeout 15 --max-time 1800 \
-            --output "$response_file" --write-out '%{http_code}' \
+            "$DOKPLOY_URL/api/application.dropDeployment"
-            "$DOKPLOY_URL/api/application.dropDeployment" 2>/dev/null || true)"
+          echo "Dokploy accepted the source archive; waiting for the VPS build and deployment."
-          case "$upload_code" in
-            2??) echo "Dokploy accepted the source archive; waiting for the VPS build and deployment." ;;
-            *) echo "Dokploy source upload failed (HTTP ${upload_code:-000}); response withheld." >&2; exit 1 ;;
-          esac
 
           attempt=0
           while [ "$attempt" -lt 360 ]; do
-            status_code="$(curl --silent --get \
+            dokploy_request "Deployment status GET /api/deployment.all (attempt $((attempt + 1))/360)" "$deployment_file" --get \
-              --header "@$auth_file" \
               --data-urlencode "applicationId=$DOKPLOY_APPLICATION_ID" \
               --connect-timeout 10 --max-time 30 \
-              --output "$deployment_file" --write-out '%{http_code}' \
+              "$DOKPLOY_URL/api/deployment.all"
-              "$DOKPLOY_URL/api/deployment.all" 2>/dev/null || true)"
-            case "$status_code" in
-              2??) ;;
-              *) echo "Dokploy deployment status check failed (HTTP ${status_code:-000})." >&2; exit 1 ;;
-            esac
 
             status="$(DOKPLOY_STATUS_FILE="$deployment_file" DOKPLOY_PREVIOUS_DEPLOYMENT_ID="$previous_id" node <<'NODE'
           const fs = require("node:fs");
-          const payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8"));
+          let payload;
+          try { payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); }
+          catch { console.error("Dokploy deployment status was not valid JSON."); process.exit(2); }
           const rows = Array.isArray(payload)
             ? payload
             : [payload?.data, payload?.deployments, payload?.items].find(Array.isArray);
@@ -117,7 +146,9 @@ jobs:
           }
           NODE
             )" || {
-              echo "Dokploy returned an unexpected deployment status; response withheld." >&2
+              node scripts/dokploy-diagnostics.mjs request "Deployment status response parsing" parse-error \
+                "$metrics_file" "$error_file" "$headers_file" "$deployment_file" "$auth_file" >&2
+              echo "Dokploy returned an unexpected deployment status; inspect the response diagnostics above." >&2
               exit 1
             }
 

docs/dokploy-deployment.md

@@ -116,10 +116,21 @@ the commit, upload it, and inspect deployment status. Gitea documents how
 runner labels select host versus Docker execution in the [runner
 documentation](https://gitea.com/gitea/runner/src/branch/main/README.md#labels).
 
-Every push to `master` triggers a deployment. The job suppresses API response
+Every push to `master` triggers a deployment. The job logs the pinned target,
-bodies and infrastructure addresses in its logs, waits for Dokploy's deployment
+commit, curl/TLS runtime versions, DNS results, proxy configuration (without
-status, and fails if the upload or deployment fails. Build logs remain available
+credentials), CA configuration status, and source archive size. Each request logs
-in the Dokploy dashboard.
+curl's exit code, HTTP status, connection IPs, certificate verification result,
+transfer sizes, and DNS/TCP/TLS/first-byte/total timings. Curl errors are preserved
+and common DNS, connection, proxy, timeout and certificate failures include hints.
+
+Failed responses include selected headers, JSON error/message fields, or an HTML
+page title. API keys and application IDs are redacted; cookies, source archives,
+deployment payloads and complete response bodies are not printed. The workflow
+does not enable shell tracing, disable TLS verification, follow authenticated
+redirects, or retry source uploads automatically. HTTP `000` means no HTTP
+response was received; inspect the curl exit code and error immediately above it.
+The job waits for Dokploy's deployment status and fails if the upload or deployment
+fails. Build logs remain available in the Dokploy dashboard.
 
 `git archive` sends tracked files from that commit only. It does not include
 `.git` metadata, local environment files, untracked files, or local caches. The

scripts/dokploy-diagnostics.mjs

@@ -0,0 +1,176 @@
+import fs from "node:fs";
+import dns from "node:dns/promises";
+import { pathToFileURL } from "node:url";
+
+export function redact(value, secrets = []) {
+  let text = String(value);
+  for (const secret of secrets.filter(Boolean)) {
+    for (const variant of new Set([secret, encodeURIComponent(secret)]))
+      text = text.split(variant).join("[REDACTED]");
+  }
+  return text
+    .replace(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/gi, "$1[REDACTED]@")
+    .replace(
+      /((?:authorization|x-api-key|cookie|set-cookie)\s*:\s*)[^\r\n]*/gi,
+      "$1[REDACTED]",
+    )
+    .replace(/\b(Bearer|Basic)\s+[^\s"<>]+/gi, "$1 [REDACTED]")
+    .replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, "");
+}
+
+export function responseDetails(headers, body) {
+  const lines = headers
+    .split(/\r?\n/)
+    .filter((line) =>
+      /^(HTTP\/|content-type:|content-length:|server:|location:|cf-ray:|x-request-id:|retry-after:)/i.test(
+        line,
+      ),
+    );
+  if (!body) return [...lines, "Response body: empty"];
+  try {
+    const data = JSON.parse(body);
+    // Deployment lists may contain source/configuration. Never dump the payload.
+    lines.push(
+      `Response JSON shape: ${Array.isArray(data) ? `array (${data.length} items)` : data && typeof data === "object" ? Object.keys(data).slice(0, 20).join(", ") : typeof data}`,
+    );
+    for (const field of ["message", "error", "code"]) {
+      const value = data?.[field];
+      if (typeof value === "string" || typeof value === "number")
+        lines.push(`${field}: ${String(value).slice(0, 2000)}`);
+      else if (value && typeof value === "object") {
+        for (const key of ["message", "code"])
+          if (typeof value[key] === "string" || typeof value[key] === "number")
+            lines.push(`${field}.${key}: ${String(value[key]).slice(0, 2000)}`);
+      }
+    }
+  } catch {
+    const title = body.match(/<title[^>]*>([^<]{0,300})<\/title>/i)?.[1];
+    lines.push(
+      title
+        ? `Response HTML title: ${title}`
+        : `Response is not JSON (${Buffer.byteLength(body)} bytes); raw content withheld.`,
+    );
+  }
+  return lines;
+}
+
+async function main([mode, ...args]) {
+  if (mode === "environment") {
+    const url = new URL(args[0]);
+    console.error(
+      `[Dokploy] Target: ${url.origin}; Node ${process.version}; runner ${process.platform}/${process.arch}`,
+    );
+    for (const name of [
+      "HTTPS_PROXY",
+      "https_proxy",
+      "HTTP_PROXY",
+      "http_proxy",
+      "ALL_PROXY",
+      "all_proxy",
+    ]) {
+      if (!process.env[name]) continue;
+      try {
+        const proxy = new URL(process.env[name]);
+        console.error(
+          `[Dokploy] ${name}: ${proxy.protocol}//${proxy.host} (credentials omitted)`,
+        );
+      } catch {
+        console.error(`[Dokploy] ${name}: configured (value withheld)`);
+      }
+    }
+    console.error(
+      `[Dokploy] NO_PROXY/no_proxy: ${process.env.NO_PROXY || process.env.no_proxy ? "configured" : "unset"}`,
+    );
+    for (const name of ["CURL_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR"])
+      console.error(
+        `[Dokploy] ${name}: ${process.env[name] ? `configured; path exists=${fs.existsSync(process.env[name])}` : "default"}`,
+      );
+    let timer;
+    try {
+      const addresses = await Promise.race([
+        dns.lookup(url.hostname, { all: true }),
+        new Promise((_, reject) => {
+          timer = setTimeout(
+            () => reject(new Error("DNS lookup exceeded 5 seconds")),
+            5000,
+          );
+        }),
+      ]);
+      console.error(
+        `[Dokploy] DNS ${url.hostname}: ${addresses.map((a) => `IPv${a.family} ${a.address}`).join(", ") || "no addresses"}`,
+      );
+    } catch (error) {
+      console.error(
+        `[Dokploy] DNS lookup failed: ${error.code ?? ""} ${error.message}`,
+      );
+    } finally {
+      clearTimeout(timer);
+    }
+    return;
+  }
+  if (mode !== "request")
+    throw new Error("Expected environment or request mode");
+  const [
+    label,
+    exitCode,
+    metricsPath,
+    errorPath,
+    headersPath,
+    bodyPath,
+    authPath,
+  ] = args;
+  const read = (file) =>
+    fs.existsSync(file) ? fs.readFileSync(file, "utf8") : "";
+  const key = read(authPath)
+    .replace(/^x-api-key:\s*/i, "")
+    .trim();
+  const safe = (text) =>
+    redact(text, [key, process.env.DOKPLOY_APPLICATION_ID]);
+  const metrics = read(metricsPath).trim();
+  const status = metrics.split(/\r?\n/)[0] || "000";
+  console.error(
+    safe(
+      `[Dokploy] ${label}: curl exit=${exitCode}\n${metrics || "No transfer metrics returned"}`,
+    ),
+  );
+  const stderr = read(errorPath).trim();
+  if (stderr)
+    console.error(
+      safe(`[Dokploy] curl diagnostics:\n${stderr.slice(0, 8000)}`),
+    );
+  const hints = {
+    5: "Proxy DNS failed. Check proxy configuration in the runner container.",
+    6: "Target DNS failed. Check the domain records and runner container DNS.",
+    7: "Connection failed. Check port 443, reverse proxy, firewall and runner routing.",
+    28: "Request timed out. Transfer timings show whether DNS, connection or the server stalled.",
+    35: "TLS handshake failed. Check the reverse proxy HTTPS listener and TLS configuration.",
+    52: "The server closed the connection without an HTTP response.",
+    60: "TLS certificate verification failed. Check the domain certificate, intermediate chain and runner CA bundle.",
+    97: "Proxy handshake failed. Check the runner proxy configuration.",
+  };
+  if (hints[exitCode]) console.error(`[Dokploy] ${hints[exitCode]}`);
+  if (exitCode !== "0" || !/^2\d\d$/.test(status)) {
+    for (const line of responseDetails(read(headersPath), read(bodyPath)))
+      console.error(safe(`[Dokploy] ${line}`));
+    if (status === "000")
+      console.error(
+        "[Dokploy] HTTP 000 means no HTTP response was received; inspect the curl error and network timings above.",
+      );
+    if (/^3\d\d$/.test(status))
+      console.error(
+        "[Dokploy] Unexpected redirect. Check the Location header and proxy/auth routing; authenticated requests do not follow redirects.",
+      );
+    if (["401", "403"].includes(status))
+      console.error(
+        "[Dokploy] Access rejected by Dokploy or a proxy. Check API key permissions and proxy access rules.",
+      );
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
+  main(process.argv.slice(2))
+    .then(() => process.exit(0))
+    .catch((error) => {
+      console.error(`[Dokploy] Diagnostics failed: ${error.message}`);
+      process.exit(1);
+    });

tests/dokploy-diagnostics.test.ts

@@ -0,0 +1,205 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync, type SpawnSyncReturns } from "node:child_process";
+import { redact, responseDetails } from "../scripts/dokploy-diagnostics.mjs";
+
+test("Dokploy diagnostics redact credentials and strip terminal controls", () => {
+  const key = "secret/key?value";
+  const message = redact(
+    `Key ${key} encoded ${encodeURIComponent(key)}\nx-api-key: ${key}\nAuthorization: Bearer token\nhttps://user:password@proxy.test\n\u001b[31m`,
+    [key],
+  );
+  for (const value of [
+    key,
+    encodeURIComponent(key),
+    "Bearer token",
+    "user:password",
+    "\u001b",
+  ])
+    assert.ok(!message.includes(value));
+  assert.ok(message.includes("[REDACTED]"));
+});
+
+test("API failure details include useful error fields without dumping deployments, configuration or cookies", () => {
+  const result = responseDetails(
+    "HTTP/2 403\r\nserver: cloudflare\r\ncf-ray: abc\r\nset-cookie: private\r\n",
+    JSON.stringify({
+      error: {
+        message: "Permission denied",
+        code: "FORBIDDEN",
+        config: "private config",
+      },
+      source: "private source",
+    }),
+  ).join("\n");
+  assert.ok(result.includes("HTTP/2 403"));
+  assert.ok(result.includes("Permission denied"));
+  assert.ok(result.includes("cf-ray: abc"));
+  for (const value of ["private config", "private source", "set-cookie"])
+    assert.ok(!result.includes(value));
+  const html = responseDetails(
+    "content-type: text/html",
+    "<title>Bad Gateway</title><body>Private proxy details</body>",
+  ).join("\n");
+  assert.ok(html.includes("Bad Gateway"));
+  assert.ok(!html.includes("Private proxy details"));
+});
+
+test("HTTP 000 diagnostics preserve curl DNS/TLS errors, timings and exit codes while masking the key", () => {
+  const directory = fs.mkdtempSync(
+    path.join(os.tmpdir(), "minegeo-dokploy-diagnostics-"),
+  );
+  try {
+    const file = (name: string, value: string) => {
+      const target = path.join(directory, name);
+      fs.writeFileSync(target, value);
+      return target;
+    };
+    const key = "test-dokploy-secret-123";
+    const metrics = file(
+      "metrics",
+      "000\nremote_ip= dns=0.002s connect=0s tls=0s total=0.003s\nssl_verify_result=20\n",
+    );
+    const headers = file("headers", "");
+    const body = file("body", "");
+    const auth = file("auth", `x-api-key: ${key}\n`);
+    for (const [code, error, expected] of [
+      [
+        "6",
+        "curl: (6) Could not resolve host: dokploy.minegeo.site",
+        "Target DNS failed",
+      ],
+      [
+        "60",
+        `curl: (60) SSL certificate problem: unable to get local issuer certificate ${key}`,
+        "TLS certificate verification failed",
+      ],
+    ]) {
+      const stderr = file("error", error);
+      const result = spawnSync(
+        process.execPath,
+        [
+          "scripts/dokploy-diagnostics.mjs",
+          "request",
+          "API preflight",
+          code,
+          metrics,
+          stderr,
+          headers,
+          body,
+          auth,
+        ],
+        { encoding: "utf8" },
+      );
+      assert.equal(result.status, 0, result.stderr);
+      assert.ok(result.stderr.includes(`curl exit=${code}`));
+      assert.ok(result.stderr.includes(expected));
+      assert.ok(result.stderr.includes("HTTP 000 means no HTTP response"));
+      assert.ok(result.stderr.includes("dns=0.002s"));
+      assert.ok(result.stderr.includes("Response body: empty"));
+      assert.ok(!result.stderr.includes(key));
+      assert.equal(result.stdout, "");
+    }
+  } finally {
+    fs.rmSync(directory, { recursive: true });
+  }
+});
+
+test("deployment workflow has valid Bash syntax and captures failures without disabling certificate checks", () => {
+  const workflow = fs
+    .readFileSync(".gitea/workflows/deploy-dokploy.yml", "utf8")
+    .replace(/\r\n/g, "\n");
+  const script = workflow
+    .split("        run: |\n")[1]
+    ?.split("\n")
+    .map((line) => line.replace(/^          /, ""))
+    .join("\n");
+  assert.ok(script, "workflow script was not found");
+  assert.ok(
+    script.includes("readonly DOKPLOY_URL='https://dokploy.minegeo.site'"),
+  );
+  assert.ok(script.includes("curl_exit=$?"));
+  assert.ok(script.includes("--silent --show-error"));
+  assert.ok(!script.includes("2>/dev/null"));
+  assert.ok(!script.includes("--insecure"));
+  const bash =
+    process.platform === "win32"
+      ? "C:/Program Files/Git/bin/bash.exe"
+      : "/bin/bash";
+  const result = spawnSync(bash, ["--noprofile", "--norc", "-n"], {
+    input: script,
+    encoding: "utf8",
+  });
+  assert.equal(result.status, 0, result.stderr || String(result.error));
+});
+
+test("the workflow request wrapper fails on HTTP 000 and reports the original curl exit code", () => {
+  const workflow = fs
+    .readFileSync(".gitea/workflows/deploy-dokploy.yml", "utf8")
+    .replace(/\r\n/g, "\n");
+  const script = workflow
+    .split("        run: |\n")[1]
+    .split("\n")
+    .map((line) => line.replace(/^          /, ""))
+    .join("\n");
+  const wrapper = script.match(/dokploy_request\(\) \{[\s\S]*?\n\}/)?.[0];
+  assert.ok(wrapper);
+  const directory = fs.mkdtempSync(
+    path.join(os.tmpdir(), "minegeo-dokploy-wrapper-"),
+  );
+  const bash =
+    process.platform === "win32"
+      ? "C:/Program Files/Git/bin/bash.exe"
+      : "/bin/bash";
+  try {
+    fs.writeFileSync(
+      path.join(directory, "auth"),
+      "x-api-key: fake-secret-for-wrapper-test\n",
+    );
+    for (const [code, http, expected] of [
+      [6, "000", 1],
+      [60, "000", 1],
+      [0, "403", 1],
+      [0, "200", 0],
+    ] as const) {
+      const result: SpawnSyncReturns<string> = spawnSync(
+        bash,
+        ["--noprofile", "--norc", "-s"],
+        {
+          encoding: "utf8",
+          env: {
+            ...process.env,
+            FIXTURE_DIR: directory.replace(/\\/g, "/"),
+            DOKPLOY_APPLICATION_ID: "test-app",
+          },
+          input: `set -eu
+auth_file="$FIXTURE_DIR/auth"
+metrics_file="$FIXTURE_DIR/metrics"
+error_file="$FIXTURE_DIR/error"
+headers_file="$FIXTURE_DIR/headers"
+${wrapper}
+curl() { printf '${http}\\ndns=0.001s total=0.002s\\n'; if [ ${code} -ne 0 ]; then printf 'curl: (${code}) simulated network failure\\n' >&2; fi; return ${code}; }
+dokploy_request 'Mock API preflight' "$FIXTURE_DIR/body" --get https://dokploy.minegeo.site/api/deployment.all
+`,
+        },
+      );
+      assert.equal(
+        result.status,
+        expected,
+        result.stderr || String(result.error),
+      );
+      assert.ok(result.stderr.includes(`curl exit=${code}`));
+      assert.ok(result.stderr.includes(http));
+      assert.ok(!result.stderr.includes("fake-secret-for-wrapper-test"));
+      if (code)
+        assert.ok(
+          result.stderr.includes(`curl: (${code}) simulated network failure`),
+        );
+    }
+  } finally {
+    fs.rmSync(directory, { recursive: true });
+  }
+});