diff --git a/.gitea/workflows/deploy-dokploy.yml b/.gitea/workflows/deploy-dokploy.yml index ad0a9f7..19774c9 100644 --- a/.gitea/workflows/deploy-dokploy.yml +++ b/.gitea/workflows/deploy-dokploy.yml @@ -33,8 +33,11 @@ jobs: archive_file="$(mktemp "$temp_root/minegeo-dokploy-source.XXXXXX")" response_file="$(mktemp "$temp_root/minegeo-dokploy-response.XXXXXX")" deployment_file="$(mktemp "$temp_root/minegeo-dokploy-status.XXXXXX")" + metrics_file="$(mktemp "$temp_root/minegeo-dokploy-metrics.XXXXXX")" + error_file="$(mktemp "$temp_root/minegeo-dokploy-errors.XXXXXX")" + headers_file="$(mktemp "$temp_root/minegeo-dokploy-headers.XXXXXX")" cleanup() { - rm -f "$auth_file" "$archive_file" "$response_file" "$deployment_file" + rm -f "$auth_file" "$archive_file" "$response_file" "$deployment_file" "$metrics_file" "$error_file" "$headers_file" } trap cleanup EXIT @@ -42,24 +45,57 @@ jobs: printf 'x-api-key: %s\n' "$DOKPLOY_API_KEY" > "$auth_file" unset DOKPLOY_API_KEY + echo "[Dokploy] Workflow revision: $(git rev-parse HEAD)" + curl --version | head -n 2 + node scripts/dokploy-diagnostics.mjs environment "$DOKPLOY_URL" + + dokploy_request() { + label="$1" + body_file="$2" + shift 2 + : > "$body_file" + : > "$headers_file" + : > "$metrics_file" + : > "$error_file" + echo "[Dokploy] Starting $label at $(date -u '+%Y-%m-%dT%H:%M:%SZ')" >&2 + if curl --silent --show-error \ + --header "@$auth_file" \ + --dump-header "$headers_file" \ + --output "$body_file" \ + --write-out '%{http_code}\nremote_ip=%{remote_ip} remote_port=%{remote_port} local_ip=%{local_ip}\ndns=%{time_namelookup}s connect=%{time_connect}s tls=%{time_appconnect}s first_byte=%{time_starttransfer}s total=%{time_total}s\nssl_verify_result=%{ssl_verify_result} redirects=%{num_redirects} uploaded=%{size_upload}B downloaded=%{size_download}B\n' \ + "$@" > "$metrics_file" 2> "$error_file"; then + curl_exit=0 + else + curl_exit=$? + fi + node scripts/dokploy-diagnostics.mjs request "$label" "$curl_exit" \ + "$metrics_file" "$error_file" "$headers_file" "$body_file" "$auth_file" >&2 + http_code="$(head -n 1 "$metrics_file")" + if [ "$curl_exit" -ne 0 ]; then + echo "[Dokploy] $label failed: curl exit $curl_exit, HTTP ${http_code:-000}." >&2 + return 1 + fi + case "$http_code" in + 2??) ;; + *) echo "[Dokploy] $label failed: HTTP ${http_code:-000}." >&2; return 1 ;; + esac + } + # Only committed files from this revision are uploaded. This excludes # .git metadata, local .env files, caches, and untracked workspace data. git archive --format=zip --output="$archive_file" HEAD + echo "[Dokploy] Source archive size: $(wc -c < "$archive_file") bytes" - before_code="$(curl --silent --get \ - --header "@$auth_file" \ + dokploy_request "API preflight GET /api/deployment.all" "$deployment_file" --get \ --data-urlencode "applicationId=$DOKPLOY_APPLICATION_ID" \ --connect-timeout 10 --max-time 30 \ - --output "$deployment_file" --write-out '%{http_code}' \ - "$DOKPLOY_URL/api/deployment.all" 2>/dev/null || true)" - case "$before_code" in - 2??) ;; - *) echo "Dokploy API preflight failed (HTTP ${before_code:-000}); response withheld." >&2; exit 1 ;; - esac + "$DOKPLOY_URL/api/deployment.all" previous_id="$(DOKPLOY_STATUS_FILE="$deployment_file" node <<'NODE' const fs = require("node:fs"); - const payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); + let payload; + try { payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); } + catch { console.error("Dokploy deployment list was not valid JSON."); process.exit(2); } const rows = Array.isArray(payload) ? payload : [payload?.data, payload?.deployments, payload?.items].find(Array.isArray); @@ -68,38 +104,31 @@ jobs: process.stdout.write(String(latest?.id ?? latest?.deploymentId ?? "")); NODE )" || { + node scripts/dokploy-diagnostics.mjs request "API preflight response parsing" parse-error \ + "$metrics_file" "$error_file" "$headers_file" "$deployment_file" "$auth_file" >&2 echo "Dokploy returned an unexpected deployment list; source was not uploaded." >&2 exit 1 } - upload_code="$(curl --silent \ - --header "@$auth_file" \ + dokploy_request "Source upload POST /api/application.dropDeployment" "$response_file" \ --form-string "applicationId=$DOKPLOY_APPLICATION_ID" \ --form "zip=@$archive_file;type=application/zip" \ --connect-timeout 15 --max-time 1800 \ - --output "$response_file" --write-out '%{http_code}' \ - "$DOKPLOY_URL/api/application.dropDeployment" 2>/dev/null || true)" - case "$upload_code" in - 2??) echo "Dokploy accepted the source archive; waiting for the VPS build and deployment." ;; - *) echo "Dokploy source upload failed (HTTP ${upload_code:-000}); response withheld." >&2; exit 1 ;; - esac + "$DOKPLOY_URL/api/application.dropDeployment" + echo "Dokploy accepted the source archive; waiting for the VPS build and deployment." attempt=0 while [ "$attempt" -lt 360 ]; do - status_code="$(curl --silent --get \ - --header "@$auth_file" \ + dokploy_request "Deployment status GET /api/deployment.all (attempt $((attempt + 1))/360)" "$deployment_file" --get \ --data-urlencode "applicationId=$DOKPLOY_APPLICATION_ID" \ --connect-timeout 10 --max-time 30 \ - --output "$deployment_file" --write-out '%{http_code}' \ - "$DOKPLOY_URL/api/deployment.all" 2>/dev/null || true)" - case "$status_code" in - 2??) ;; - *) echo "Dokploy deployment status check failed (HTTP ${status_code:-000})." >&2; exit 1 ;; - esac + "$DOKPLOY_URL/api/deployment.all" status="$(DOKPLOY_STATUS_FILE="$deployment_file" DOKPLOY_PREVIOUS_DEPLOYMENT_ID="$previous_id" node <<'NODE' const fs = require("node:fs"); - const payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); + let payload; + try { payload = JSON.parse(fs.readFileSync(process.env.DOKPLOY_STATUS_FILE, "utf8")); } + catch { console.error("Dokploy deployment status was not valid JSON."); process.exit(2); } const rows = Array.isArray(payload) ? payload : [payload?.data, payload?.deployments, payload?.items].find(Array.isArray); @@ -117,7 +146,9 @@ jobs: } NODE )" || { - echo "Dokploy returned an unexpected deployment status; response withheld." >&2 + node scripts/dokploy-diagnostics.mjs request "Deployment status response parsing" parse-error \ + "$metrics_file" "$error_file" "$headers_file" "$deployment_file" "$auth_file" >&2 + echo "Dokploy returned an unexpected deployment status; inspect the response diagnostics above." >&2 exit 1 } diff --git a/docs/dokploy-deployment.md b/docs/dokploy-deployment.md index 4890c89..a39faad 100644 --- a/docs/dokploy-deployment.md +++ b/docs/dokploy-deployment.md @@ -116,10 +116,21 @@ the commit, upload it, and inspect deployment status. Gitea documents how runner labels select host versus Docker execution in the [runner documentation](https://gitea.com/gitea/runner/src/branch/main/README.md#labels). -Every push to `master` triggers a deployment. The job suppresses API response -bodies and infrastructure addresses in its logs, waits for Dokploy's deployment -status, and fails if the upload or deployment fails. Build logs remain available -in the Dokploy dashboard. +Every push to `master` triggers a deployment. The job logs the pinned target, +commit, curl/TLS runtime versions, DNS results, proxy configuration (without +credentials), CA configuration status, and source archive size. Each request logs +curl's exit code, HTTP status, connection IPs, certificate verification result, +transfer sizes, and DNS/TCP/TLS/first-byte/total timings. Curl errors are preserved +and common DNS, connection, proxy, timeout and certificate failures include hints. + +Failed responses include selected headers, JSON error/message fields, or an HTML +page title. API keys and application IDs are redacted; cookies, source archives, +deployment payloads and complete response bodies are not printed. The workflow +does not enable shell tracing, disable TLS verification, follow authenticated +redirects, or retry source uploads automatically. HTTP `000` means no HTTP +response was received; inspect the curl exit code and error immediately above it. +The job waits for Dokploy's deployment status and fails if the upload or deployment +fails. Build logs remain available in the Dokploy dashboard. `git archive` sends tracked files from that commit only. It does not include `.git` metadata, local environment files, untracked files, or local caches. The diff --git a/scripts/dokploy-diagnostics.mjs b/scripts/dokploy-diagnostics.mjs new file mode 100644 index 0000000..18e9ce6 --- /dev/null +++ b/scripts/dokploy-diagnostics.mjs @@ -0,0 +1,176 @@ +import fs from "node:fs"; +import dns from "node:dns/promises"; +import { pathToFileURL } from "node:url"; + +export function redact(value, secrets = []) { + let text = String(value); + for (const secret of secrets.filter(Boolean)) { + for (const variant of new Set([secret, encodeURIComponent(secret)])) + text = text.split(variant).join("[REDACTED]"); + } + return text + .replace(/(https?:\/\/)[^\s/@]+:[^\s/@]+@/gi, "$1[REDACTED]@") + .replace( + /((?:authorization|x-api-key|cookie|set-cookie)\s*:\s*)[^\r\n]*/gi, + "$1[REDACTED]", + ) + .replace(/\b(Bearer|Basic)\s+[^\s"<>]+/gi, "$1 [REDACTED]") + .replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, ""); +} + +export function responseDetails(headers, body) { + const lines = headers + .split(/\r?\n/) + .filter((line) => + /^(HTTP\/|content-type:|content-length:|server:|location:|cf-ray:|x-request-id:|retry-after:)/i.test( + line, + ), + ); + if (!body) return [...lines, "Response body: empty"]; + try { + const data = JSON.parse(body); + // Deployment lists may contain source/configuration. Never dump the payload. + lines.push( + `Response JSON shape: ${Array.isArray(data) ? `array (${data.length} items)` : data && typeof data === "object" ? Object.keys(data).slice(0, 20).join(", ") : typeof data}`, + ); + for (const field of ["message", "error", "code"]) { + const value = data?.[field]; + if (typeof value === "string" || typeof value === "number") + lines.push(`${field}: ${String(value).slice(0, 2000)}`); + else if (value && typeof value === "object") { + for (const key of ["message", "code"]) + if (typeof value[key] === "string" || typeof value[key] === "number") + lines.push(`${field}.${key}: ${String(value[key]).slice(0, 2000)}`); + } + } + } catch { + const title = body.match(/]*>([^<]{0,300})<\/title>/i)?.[1]; + lines.push( + title + ? `Response HTML title: ${title}` + : `Response is not JSON (${Buffer.byteLength(body)} bytes); raw content withheld.`, + ); + } + return lines; +} + +async function main([mode, ...args]) { + if (mode === "environment") { + const url = new URL(args[0]); + console.error( + `[Dokploy] Target: ${url.origin}; Node ${process.version}; runner ${process.platform}/${process.arch}`, + ); + for (const name of [ + "HTTPS_PROXY", + "https_proxy", + "HTTP_PROXY", + "http_proxy", + "ALL_PROXY", + "all_proxy", + ]) { + if (!process.env[name]) continue; + try { + const proxy = new URL(process.env[name]); + console.error( + `[Dokploy] ${name}: ${proxy.protocol}//${proxy.host} (credentials omitted)`, + ); + } catch { + console.error(`[Dokploy] ${name}: configured (value withheld)`); + } + } + console.error( + `[Dokploy] NO_PROXY/no_proxy: ${process.env.NO_PROXY || process.env.no_proxy ? "configured" : "unset"}`, + ); + for (const name of ["CURL_CA_BUNDLE", "SSL_CERT_FILE", "SSL_CERT_DIR"]) + console.error( + `[Dokploy] ${name}: ${process.env[name] ? `configured; path exists=${fs.existsSync(process.env[name])}` : "default"}`, + ); + let timer; + try { + const addresses = await Promise.race([ + dns.lookup(url.hostname, { all: true }), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("DNS lookup exceeded 5 seconds")), + 5000, + ); + }), + ]); + console.error( + `[Dokploy] DNS ${url.hostname}: ${addresses.map((a) => `IPv${a.family} ${a.address}`).join(", ") || "no addresses"}`, + ); + } catch (error) { + console.error( + `[Dokploy] DNS lookup failed: ${error.code ?? ""} ${error.message}`, + ); + } finally { + clearTimeout(timer); + } + return; + } + if (mode !== "request") + throw new Error("Expected environment or request mode"); + const [ + label, + exitCode, + metricsPath, + errorPath, + headersPath, + bodyPath, + authPath, + ] = args; + const read = (file) => + fs.existsSync(file) ? fs.readFileSync(file, "utf8") : ""; + const key = read(authPath) + .replace(/^x-api-key:\s*/i, "") + .trim(); + const safe = (text) => + redact(text, [key, process.env.DOKPLOY_APPLICATION_ID]); + const metrics = read(metricsPath).trim(); + const status = metrics.split(/\r?\n/)[0] || "000"; + console.error( + safe( + `[Dokploy] ${label}: curl exit=${exitCode}\n${metrics || "No transfer metrics returned"}`, + ), + ); + const stderr = read(errorPath).trim(); + if (stderr) + console.error( + safe(`[Dokploy] curl diagnostics:\n${stderr.slice(0, 8000)}`), + ); + const hints = { + 5: "Proxy DNS failed. Check proxy configuration in the runner container.", + 6: "Target DNS failed. Check the domain records and runner container DNS.", + 7: "Connection failed. Check port 443, reverse proxy, firewall and runner routing.", + 28: "Request timed out. Transfer timings show whether DNS, connection or the server stalled.", + 35: "TLS handshake failed. Check the reverse proxy HTTPS listener and TLS configuration.", + 52: "The server closed the connection without an HTTP response.", + 60: "TLS certificate verification failed. Check the domain certificate, intermediate chain and runner CA bundle.", + 97: "Proxy handshake failed. Check the runner proxy configuration.", + }; + if (hints[exitCode]) console.error(`[Dokploy] ${hints[exitCode]}`); + if (exitCode !== "0" || !/^2\d\d$/.test(status)) { + for (const line of responseDetails(read(headersPath), read(bodyPath))) + console.error(safe(`[Dokploy] ${line}`)); + if (status === "000") + console.error( + "[Dokploy] HTTP 000 means no HTTP response was received; inspect the curl error and network timings above.", + ); + if (/^3\d\d$/.test(status)) + console.error( + "[Dokploy] Unexpected redirect. Check the Location header and proxy/auth routing; authenticated requests do not follow redirects.", + ); + if (["401", "403"].includes(status)) + console.error( + "[Dokploy] Access rejected by Dokploy or a proxy. Check API key permissions and proxy access rules.", + ); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) + main(process.argv.slice(2)) + .then(() => process.exit(0)) + .catch((error) => { + console.error(`[Dokploy] Diagnostics failed: ${error.message}`); + process.exit(1); + }); diff --git a/tests/dokploy-diagnostics.test.ts b/tests/dokploy-diagnostics.test.ts new file mode 100644 index 0000000..5c92adf --- /dev/null +++ b/tests/dokploy-diagnostics.test.ts @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync, type SpawnSyncReturns } from "node:child_process"; +import { redact, responseDetails } from "../scripts/dokploy-diagnostics.mjs"; + +test("Dokploy diagnostics redact credentials and strip terminal controls", () => { + const key = "secret/key?value"; + const message = redact( + `Key ${key} encoded ${encodeURIComponent(key)}\nx-api-key: ${key}\nAuthorization: Bearer token\nhttps://user:password@proxy.test\n\u001b[31m`, + [key], + ); + for (const value of [ + key, + encodeURIComponent(key), + "Bearer token", + "user:password", + "\u001b", + ]) + assert.ok(!message.includes(value)); + assert.ok(message.includes("[REDACTED]")); +}); + +test("API failure details include useful error fields without dumping deployments, configuration or cookies", () => { + const result = responseDetails( + "HTTP/2 403\r\nserver: cloudflare\r\ncf-ray: abc\r\nset-cookie: private\r\n", + JSON.stringify({ + error: { + message: "Permission denied", + code: "FORBIDDEN", + config: "private config", + }, + source: "private source", + }), + ).join("\n"); + assert.ok(result.includes("HTTP/2 403")); + assert.ok(result.includes("Permission denied")); + assert.ok(result.includes("cf-ray: abc")); + for (const value of ["private config", "private source", "set-cookie"]) + assert.ok(!result.includes(value)); + const html = responseDetails( + "content-type: text/html", + "Bad GatewayPrivate proxy details", + ).join("\n"); + assert.ok(html.includes("Bad Gateway")); + assert.ok(!html.includes("Private proxy details")); +}); + +test("HTTP 000 diagnostics preserve curl DNS/TLS errors, timings and exit codes while masking the key", () => { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), "minegeo-dokploy-diagnostics-"), + ); + try { + const file = (name: string, value: string) => { + const target = path.join(directory, name); + fs.writeFileSync(target, value); + return target; + }; + const key = "test-dokploy-secret-123"; + const metrics = file( + "metrics", + "000\nremote_ip= dns=0.002s connect=0s tls=0s total=0.003s\nssl_verify_result=20\n", + ); + const headers = file("headers", ""); + const body = file("body", ""); + const auth = file("auth", `x-api-key: ${key}\n`); + for (const [code, error, expected] of [ + [ + "6", + "curl: (6) Could not resolve host: dokploy.minegeo.site", + "Target DNS failed", + ], + [ + "60", + `curl: (60) SSL certificate problem: unable to get local issuer certificate ${key}`, + "TLS certificate verification failed", + ], + ]) { + const stderr = file("error", error); + const result = spawnSync( + process.execPath, + [ + "scripts/dokploy-diagnostics.mjs", + "request", + "API preflight", + code, + metrics, + stderr, + headers, + body, + auth, + ], + { encoding: "utf8" }, + ); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stderr.includes(`curl exit=${code}`)); + assert.ok(result.stderr.includes(expected)); + assert.ok(result.stderr.includes("HTTP 000 means no HTTP response")); + assert.ok(result.stderr.includes("dns=0.002s")); + assert.ok(result.stderr.includes("Response body: empty")); + assert.ok(!result.stderr.includes(key)); + assert.equal(result.stdout, ""); + } + } finally { + fs.rmSync(directory, { recursive: true }); + } +}); + +test("deployment workflow has valid Bash syntax and captures failures without disabling certificate checks", () => { + const workflow = fs + .readFileSync(".gitea/workflows/deploy-dokploy.yml", "utf8") + .replace(/\r\n/g, "\n"); + const script = workflow + .split(" run: |\n")[1] + ?.split("\n") + .map((line) => line.replace(/^ /, "")) + .join("\n"); + assert.ok(script, "workflow script was not found"); + assert.ok( + script.includes("readonly DOKPLOY_URL='https://dokploy.minegeo.site'"), + ); + assert.ok(script.includes("curl_exit=$?")); + assert.ok(script.includes("--silent --show-error")); + assert.ok(!script.includes("2>/dev/null")); + assert.ok(!script.includes("--insecure")); + const bash = + process.platform === "win32" + ? "C:/Program Files/Git/bin/bash.exe" + : "/bin/bash"; + const result = spawnSync(bash, ["--noprofile", "--norc", "-n"], { + input: script, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr || String(result.error)); +}); + +test("the workflow request wrapper fails on HTTP 000 and reports the original curl exit code", () => { + const workflow = fs + .readFileSync(".gitea/workflows/deploy-dokploy.yml", "utf8") + .replace(/\r\n/g, "\n"); + const script = workflow + .split(" run: |\n")[1] + .split("\n") + .map((line) => line.replace(/^ /, "")) + .join("\n"); + const wrapper = script.match(/dokploy_request\(\) \{[\s\S]*?\n\}/)?.[0]; + assert.ok(wrapper); + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), "minegeo-dokploy-wrapper-"), + ); + const bash = + process.platform === "win32" + ? "C:/Program Files/Git/bin/bash.exe" + : "/bin/bash"; + try { + fs.writeFileSync( + path.join(directory, "auth"), + "x-api-key: fake-secret-for-wrapper-test\n", + ); + for (const [code, http, expected] of [ + [6, "000", 1], + [60, "000", 1], + [0, "403", 1], + [0, "200", 0], + ] as const) { + const result: SpawnSyncReturns = spawnSync( + bash, + ["--noprofile", "--norc", "-s"], + { + encoding: "utf8", + env: { + ...process.env, + FIXTURE_DIR: directory.replace(/\\/g, "/"), + DOKPLOY_APPLICATION_ID: "test-app", + }, + input: `set -eu +auth_file="$FIXTURE_DIR/auth" +metrics_file="$FIXTURE_DIR/metrics" +error_file="$FIXTURE_DIR/error" +headers_file="$FIXTURE_DIR/headers" +${wrapper} +curl() { printf '${http}\\ndns=0.001s total=0.002s\\n'; if [ ${code} -ne 0 ]; then printf 'curl: (${code}) simulated network failure\\n' >&2; fi; return ${code}; } +dokploy_request 'Mock API preflight' "$FIXTURE_DIR/body" --get https://dokploy.minegeo.site/api/deployment.all +`, + }, + ); + assert.equal( + result.status, + expected, + result.stderr || String(result.error), + ); + assert.ok(result.stderr.includes(`curl exit=${code}`)); + assert.ok(result.stderr.includes(http)); + assert.ok(!result.stderr.includes("fake-secret-for-wrapper-test")); + if (code) + assert.ok( + result.stderr.includes(`curl: (${code}) simulated network failure`), + ); + } + } finally { + fs.rmSync(directory, { recursive: true }); + } +});