[Init] Initial commit - NetMesh terminal manager

scripts/compare-ci-test-baseline.cjs

@@ -0,0 +1,551 @@
+'use strict';
+
+const fs = require('node:fs');
+
+const ANSI_RE = /\x1b\[[0-?]*[ -/]*[@-~]/g;
+
+// Known app/test logger tags that emit volatile `# [Tag] ...` TAP comments on
+// every npm test run. Skip these specifically; unknown bracket tags still enter
+// nonTapOutput so new runner diagnostics keep failing the gate.
+const VOLATILE_TAP_DIAG_TAG_RE = new RegExp(
+  '^# \\[(?:'
+  + [
+    'transferDiag',
+    'transferBridge',
+    'FileWatcher',
+    'SessionLogStream',
+    'SSH',
+    'SSH Exec',
+    'KeyboardInteractive',
+    'Telnet',
+    'GlobalShortcut',
+    'Chain',
+    'PortForward',
+    'PortForwardingService',
+    'SFTP',
+    'SFTP Chain',
+    'TempDir',
+    'Terminal',
+    'AutoUpdate',
+    'Passphrase',
+    'Test',
+    'resolve-mosh-bin-release',
+    'resolve-et-bin-release',
+    'KeywordHighlight',
+    'Cursor SDK',
+    'Main',
+    'vaultBackupBridge',
+    'CloudSyncManager',
+    'ZMODEM',
+    'TrayPanel',
+    'sdk',
+    'scp-it',
+    'Plugins',
+    'netcatty-mcp',
+    'DirtyEditorGuard',
+    'Credentials',
+    'afterPack',
+  ].map((tag) => tag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')
+  + ')\\]',
+);
+
+function normalizeNonTapLine(line) {
+  return String(line || '')
+    .replace(/\(node:\d+\)/g, '(node:<pid>)')
+    .replace(
+      /\b[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\b/gi,
+      '<uuid>',
+    )
+    .replace(/\bsftp-\d+\b/g, 'sftp-<id>')
+    .replace(
+      /\b(?:requestId|sessionId|transferId|id)\b(['"]?\s*[:=]\s*['"]?)[^'"\s,}\]]+/gi,
+      '$1<id>',
+    )
+    .replace(/https?:\/\/127\.0\.0\.1:\d+/g, 'http://127.0.0.1:<port>');
+}
+
+function normalizeRuntimeDetail(detail) {
+  return detail
+    .replace(
+      /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\b/g,
+      '<timestamp>',
+    )
+    .replace(/(?:\/private)?\/var\/folders\/\S+|\/tmp\/\S+/g, '<tmp-path>')
+    .replace(/:\d+:\d+\b/g, ':<line>:<column>');
+}
+
+function normalizeAssertionDetail(detail) {
+  return normalizeRuntimeDetail(detail).replace(
+    /\b(now|timestamp|pid|nonce|random(?:Value)?)\b(['"]?\s*[:=]\s*)[^,\s}\]]+/gi,
+    '$1$2<volatile>',
+  );
+}
+
+function normalizeStableDiagnosticDetail(detail) {
+  return normalizeAssertionDetail(detail);
+}
+
+function collectNonTapOutput(lines) {
+  const output = [];
+  let inDiagnostic = false;
+  let diagnosticOwnerIndent = -1;
+  let awaitingDiagnosticIndent = null;
+  for (const rawLine of lines) {
+    const line = rawLine.trim();
+    if (inDiagnostic) {
+      const indent = rawLine.match(/^\s*/)?.[0].length || 0;
+      if (/^\s+\.\.\.\s*$/.test(rawLine) && indent > diagnosticOwnerIndent) {
+        inDiagnostic = false;
+        diagnosticOwnerIndent = -1;
+        continue;
+      }
+      const boundary = line.match(
+        /^(?:(?:ok|not ok) \d+ - |# (?:tests|suites|pass|fail|cancelled|skipped|todo|duration_ms) |1\.\.\d+$)/,
+      );
+      if (!boundary || indent > diagnosticOwnerIndent) continue;
+      output.push('<unterminated-tap-diagnostic>');
+      inDiagnostic = false;
+      diagnosticOwnerIndent = -1;
+    }
+    if (awaitingDiagnosticIndent !== null) {
+      if (!line) continue;
+      const indent = rawLine.match(/^\s*/)?.[0].length || 0;
+      if (/^\s+---\s*$/.test(rawLine) && indent > awaitingDiagnosticIndent) {
+        inDiagnostic = true;
+        diagnosticOwnerIndent = awaitingDiagnosticIndent;
+        awaitingDiagnosticIndent = null;
+        continue;
+      }
+      awaitingDiagnosticIndent = null;
+    }
+    // Node's test runner emits YAML diagnostics after both ok and not ok.
+    // Treat both as attached TAP so volatile duration_ms values never look like
+    // added runner noise between baseline and candidate runs.
+    const tapResult = rawLine.match(/^(\s*)(?:ok|not ok) \d+ - /);
+    if (tapResult) {
+      awaitingDiagnosticIndent = tapResult[1].length;
+      continue;
+    }
+    if (
+      !line ||
+      /^TAP version \d+$/.test(line) ||
+      /^ok \d+ - /.test(line) ||
+      /^# (?:Subtest:|tests |suites |pass |fail |cancelled |skipped |todo |duration_ms )/.test(line) ||
+      VOLATILE_TAP_DIAG_TAG_RE.test(line) ||
+      /^1\.\.\d+$/.test(line)
+    ) {
+      continue;
+    }
+    output.push(
+      normalizeNonTapLine(
+        line
+          .replace(/\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\b/g, '<timestamp>')
+          .replace(/(?:\/private)?\/var\/folders\/\S+|\/tmp\/\S+/g, '<tmp-path>')
+          .replace(/:\d+:\d+\b/g, ':<line>:<column>'),
+      ),
+    );
+  }
+  if (inDiagnostic) output.push('<unterminated-tap-diagnostic>');
+  return output;
+}
+
+function parseTapResult(text, exitCode) {
+  const normalized = String(text || '').replace(ANSI_RE, '').replace(/\r\n?/g, '\n');
+  const lines = normalized.split('\n');
+  const failures = [];
+  const failureRecords = [];
+  const successes = [];
+  let failCount = null;
+  let cancelledCount = null;
+  let skippedCount = null;
+  let todoCount = null;
+  let testCount = null;
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    const succeeded = line.match(/^\s*ok \d+ - (.+)$/);
+    const tapDirective = /\s+#\s*(?:SKIP|TODO)\b/i;
+    if (succeeded && !tapDirective.test(line)) {
+      successes.push(succeeded[1].trim());
+    }
+    const failed = line.match(/^\s*not ok \d+ - (.+)$/);
+    if (failed && !tapDirective.test(line)) {
+      const name = failed[1].trim();
+      const diagnostic = [];
+      const errorDetails = [];
+      let inDiagnostic = false;
+      let errorBlockIndent = -1;
+      let skippingStack = false;
+      let stackIndent = -1;
+      for (let detailIndex = index + 1; detailIndex < lines.length; detailIndex += 1) {
+        const detail = lines[detailIndex];
+        if (/^\s*(?:ok|not ok) \d+ - /.test(detail) || /^\s*# (?:tests|fail|cancelled|skipped|todo) \d+\s*$/.test(detail)) {
+          break;
+        }
+        if (/^\s*---\s*$/.test(detail)) {
+          inDiagnostic = true;
+          continue;
+        }
+        if (inDiagnostic && /^\s*\.\.\.\s*$/.test(detail)) break;
+        if (!inDiagnostic) continue;
+        const indent = detail.match(/^\s*/)?.[0].length || 0;
+        if (errorBlockIndent >= 0) {
+          if (!detail.trim()) continue;
+          if (indent > errorBlockIndent) {
+            errorDetails.push(detail.trim());
+            continue;
+          }
+          errorBlockIndent = -1;
+        }
+        if (skippingStack) {
+          if (!detail.trim() || indent > stackIndent) continue;
+          skippingStack = false;
+        }
+        if (/^\s*stack:\s*(?:\|-)?\s*$/.test(detail)) {
+          skippingStack = true;
+          stackIndent = indent;
+          continue;
+        }
+        if (/^\s*duration_ms:/.test(detail)) continue;
+        if (/^\s*(?:actual|expected):/.test(detail)) continue;
+        if (/^\s*error:\s*(?:\|-|>)\s*$/.test(detail)) {
+          diagnostic.push('error:');
+          errorBlockIndent = indent;
+          continue;
+        }
+        if (detail.trim()) {
+          diagnostic.push(normalizeStableDiagnosticDetail(detail.trimEnd()));
+        }
+      }
+      const assertionFailure = diagnostic.some((detail) =>
+        /^\s*code:\s*['"]?ERR_ASSERTION['"]?\s*$/.test(detail),
+      );
+      const stableErrorDetails = assertionFailure
+        ? errorDetails.map(normalizeStableDiagnosticDetail)
+        : errorDetails.map(normalizeRuntimeDetail);
+      diagnostic.push(...stableErrorDetails.map((detail) => `error-detail: ${detail}`));
+      failures.push(name);
+      failureRecords.push({
+        name,
+        identity: diagnostic.length ? `${name}\n${diagnostic.join('\n')}` : name,
+      });
+    }
+    const failSummary = line.match(/^\s*# fail (\d+)\s*$/);
+    if (failSummary) {
+      failCount = Number(failSummary[1]);
+    }
+    const cancelledSummary = line.match(/^\s*# cancelled (\d+)\s*$/);
+    if (cancelledSummary) {
+      cancelledCount = Number(cancelledSummary[1]);
+    }
+    const skippedSummary = line.match(/^\s*# skipped (\d+)\s*$/);
+    if (skippedSummary) {
+      skippedCount = Number(skippedSummary[1]);
+    }
+    const todoSummary = line.match(/^\s*# todo (\d+)\s*$/);
+    if (todoSummary) {
+      todoCount = Number(todoSummary[1]);
+    }
+    const testSummary = line.match(/^\s*# tests (\d+)\s*$/);
+    if (testSummary) {
+      testCount = Number(testSummary[1]);
+    }
+  }
+  return {
+    exitCode: Number(exitCode),
+    failures,
+    failureRecords,
+    successes,
+    failCount,
+    cancelledCount,
+    skippedCount,
+    todoCount,
+    testCount,
+    nonTapOutput: collectNonTapOutput(lines),
+    complete:
+      failCount !== null &&
+      cancelledCount !== null &&
+      skippedCount !== null &&
+      todoCount !== null &&
+      testCount !== null,
+  };
+}
+
+function countFailures(failures) {
+  const counts = new Map();
+  for (const failure of failures) {
+    counts.set(failure, (counts.get(failure) || 0) + 1);
+  }
+  return counts;
+}
+
+function compareTapResults(baseline, candidate) {
+  const baselineSuccessCounts = countFailures(baseline.successes);
+  const candidateSuccessCounts = countFailures(candidate.successes);
+  const candidateSuccessPool = new Map(candidateSuccessCounts);
+  const missingBaselineSuccesses = [];
+  for (const [success, count] of baselineSuccessCounts) {
+    const available = candidateSuccessPool.get(success) || 0;
+    const missing = count - available;
+    candidateSuccessPool.set(success, Math.max(0, available - count));
+    for (let i = 0; i < missing; i += 1) missingBaselineSuccesses.push(success);
+  }
+  const candidateFailurePool = countFailures(
+    candidate.failureRecords.map((failure) => failure.identity),
+  );
+  const missingBaselineFailures = [];
+  for (const failure of baseline.failureRecords) {
+    const matchingFailures = candidateFailurePool.get(failure.identity) || 0;
+    if (matchingFailures > 0) {
+      candidateFailurePool.set(failure.identity, matchingFailures - 1);
+      continue;
+    }
+    const matchingSuccesses = candidateSuccessPool.get(failure.name) || 0;
+    if (matchingSuccesses > 0) {
+      candidateSuccessPool.set(failure.name, matchingSuccesses - 1);
+      continue;
+    }
+    missingBaselineFailures.push(failure.name);
+  }
+
+  const result = ({
+    passed,
+    kind,
+    newFailures = [],
+    candidateFailures = candidate.exitCode === 0 ? [] : candidate.failures,
+  }) => ({
+    passed,
+    kind,
+    baselineFailures: baseline.failures,
+    candidateFailures,
+    newFailures,
+    missingBaselineSuccesses,
+    missingBaselineFailures,
+  });
+
+  if (candidate.exitCode === 0) {
+    // Require every baseline failure title to reappear as a same-named success
+    // before accepting a red-to-green transition. Aggregate counts alone would allow
+    // deleting the failing test and adding an unrelated passer.
+    const baselineFailCounts = countFailures(baseline.failures);
+    const baselineSuccessCountsForFix = countFailures(baseline.successes);
+    const candidateSuccessCountsForFix = countFailures(candidate.successes);
+    const unresolvedBaselineFailures = [];
+    for (const [name, failCount] of baselineFailCounts) {
+      const required =
+        failCount + (baselineSuccessCountsForFix.get(name) || 0);
+      const available = candidateSuccessCountsForFix.get(name) || 0;
+      for (let i = 0; i < required - available; i += 1) {
+        unresolvedBaselineFailures.push(name);
+      }
+    }
+    // A fully green candidate may rename success titles when quantitative
+    // coverage does not shrink, but only against a complete baseline summary.
+    // An incomplete baseline cannot prove coverage was preserved, so fail closed.
+    const quantitativeClean =
+      baseline.complete &&
+      candidate.complete &&
+      candidate.failCount === 0 &&
+      candidate.cancelledCount === 0 &&
+      unresolvedBaselineFailures.length === 0 &&
+      candidate.skippedCount <= baseline.skippedCount &&
+      candidate.todoCount <= baseline.todoCount &&
+      candidate.testCount >= baseline.testCount;
+    if (quantitativeClean) {
+      return result({ passed: true, kind: 'clean' });
+    }
+    const coverageShrunk =
+      baseline.complete &&
+      candidate.complete &&
+      candidate.failCount === 0 &&
+      candidate.cancelledCount === 0 &&
+      candidate.skippedCount <= baseline.skippedCount &&
+      candidate.todoCount <= baseline.todoCount &&
+      candidate.testCount < baseline.testCount &&
+      missingBaselineSuccesses.length > 0;
+    if (coverageShrunk) {
+      return result({
+        passed: false,
+        kind: 'missing_baseline_successes',
+        newFailures: missingBaselineSuccesses,
+      });
+    }
+    return result({
+      passed: false,
+      kind: 'unclassified_failure',
+      newFailures: unresolvedBaselineFailures.length
+        ? unresolvedBaselineFailures
+        : missingBaselineSuccesses,
+    });
+  }
+
+  if (baseline.exitCode === 0) {
+    return result({
+      passed: false,
+      kind: 'new_failures',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  if (!baseline.complete || !candidate.complete) {
+    return result({
+      passed: false,
+      kind: 'unclassified_failure',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  if (missingBaselineSuccesses.length) {
+    return result({
+      passed: false,
+      kind: 'missing_baseline_successes',
+      candidateFailures: candidate.failures,
+      newFailures: missingBaselineSuccesses,
+    });
+  }
+
+  if (candidate.exitCode !== baseline.exitCode) {
+    return result({
+      passed: false,
+      kind: 'unclassified_failure',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  const baselineNonTapCounts = countFailures(baseline.nonTapOutput);
+  const candidateNonTapCounts = countFailures(candidate.nonTapOutput);
+  const addedNonTapOutput = [...candidateNonTapCounts].some(
+    ([line, count]) => count > (baselineNonTapCounts.get(line) || 0),
+  );
+  if (addedNonTapOutput) {
+    return result({
+      passed: false,
+      kind: 'unclassified_failure',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  if (candidate.failCount === 0 || candidate.testCount < baseline.testCount) {
+    return result({
+      passed: false,
+      kind: 'unclassified_failure',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  if (
+    candidate.cancelledCount > 0 ||
+    candidate.skippedCount > baseline.skippedCount ||
+    candidate.todoCount > baseline.todoCount
+  ) {
+    return result({
+      passed: false,
+      kind: 'cancelled_tests',
+      candidateFailures: candidate.failures,
+      newFailures: candidate.failures,
+    });
+  }
+
+  const baselineCounts = countFailures(
+    baseline.failureRecords.map((failure) => failure.identity),
+  );
+  const candidateCounts = countFailures(
+    candidate.failureRecords.map((failure) => failure.identity),
+  );
+  const newFailures = [];
+  for (const [identity, count] of candidateCounts) {
+    const extra = count - (baselineCounts.get(identity) || 0);
+    const name = candidate.failureRecords.find(
+      (failure) => failure.identity === identity,
+    )?.name || identity;
+    for (let i = 0; i < extra; i += 1) newFailures.push(name);
+  }
+  const parsedCountsMatch =
+    baseline.failureRecords.length >= Number(baseline.failCount) &&
+    candidate.failureRecords.length >= Number(candidate.failCount);
+  const passed =
+    newFailures.length === 0 &&
+    missingBaselineFailures.length === 0 &&
+    parsedCountsMatch;
+  const reportedFailures = newFailures.length
+    ? newFailures
+    : missingBaselineFailures;
+  return result({
+    passed,
+    kind: passed
+      ? 'baseline_only'
+      : newFailures.length
+        ? 'new_failures'
+        : 'unclassified_failure',
+    candidateFailures: candidate.failures,
+    newFailures: reportedFailures,
+  });
+}
+
+function parseArgs(argv) {
+  const values = {};
+  for (let i = 0; i < argv.length; i += 2) {
+    const key = argv[i];
+    if (!key?.startsWith('--') || argv[i + 1] === undefined) {
+      throw new Error('Expected --name value arguments.');
+    }
+    values[key.slice(2)] = argv[i + 1];
+  }
+  return values;
+}
+
+function main(argv = process.argv.slice(2)) {
+  const args = parseArgs(argv);
+  for (const required of [
+    'baseline-log',
+    'baseline-exit',
+    'candidate-log',
+    'candidate-exit',
+    'output',
+  ]) {
+    if (!(required in args)) throw new Error(`Missing --${required}.`);
+  }
+  const baseline = parseTapResult(
+    fs.readFileSync(args['baseline-log'], 'utf8'),
+    args['baseline-exit'],
+  );
+  const candidate = parseTapResult(
+    fs.readFileSync(args['candidate-log'], 'utf8'),
+    args['candidate-exit'],
+  );
+  const result = compareTapResults(baseline, candidate);
+  fs.writeFileSync(args.output, `${JSON.stringify(result, null, 2)}\n`, 'utf8');
+  console.log(
+    result.passed
+      ? `Test comparison accepted: ${result.kind}.`
+      : `Test comparison failed: ${result.kind}.`,
+  );
+  if (result.newFailures.length) {
+    console.error(`New failures:\n- ${result.newFailures.join('\n- ')}`);
+  }
+  if (!result.passed && result.missingBaselineSuccesses?.length) {
+    console.error(
+      `Missing baseline successes:\n- ${result.missingBaselineSuccesses.join('\n- ')}`,
+    );
+  }
+  return result.passed ? 0 : 1;
+}
+
+if (require.main === module) {
+  try {
+    process.exitCode = main();
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exitCode = 2;
+  }
+}
+
+module.exports = {
+  compareTapResults,
+  main,
+  parseTapResult,
+};