From 5996856792cc48bb9e516f87cd87c638ad46de04 Mon Sep 17 00:00:00 2001 From: Andrew Mikofalvy <5668128+amikofalvy@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:53:51 +0000 Subject: [PATCH 1/2] fix(ok): move project diagnostics outside synced folders (#7475) * fix(ok): move project diagnostics outside synced folders * fix(ok): retain relocated diagnostics safely across shutdown * fix(ok): keep diagnostic cleanup off spawner wait paths * fix(ok): keep MCP spawn preparation silent and isolated * fix(ok): create isolated test homes before suites run * fix(ok): preserve browser cache across test home isolation GitOrigin-RevId: fc0ef327055b1088d7a479f7ae369cffec9a9c7a --- .changeset/project-diagnostic-location.md | 7 + docs/content/reference/cli.mdx | 2 +- docs/content/reference/configuration.mdx | 4 +- .../reference/what-open-knowledge-writes.mdx | 34 +- knip.config.ts | 8 +- .../components/ConnectingBanner.dom.test.tsx | 19 +- .../app/src/components/ConnectingBanner.tsx | 5 +- packages/app/src/locales/ar/messages.po | 4 +- packages/app/src/locales/bn/messages.po | 4 +- packages/app/src/locales/en/messages.po | 4 +- packages/app/src/locales/es/messages.po | 4 +- packages/app/src/locales/fr/messages.po | 4 +- packages/app/src/locales/hi/messages.po | 4 +- packages/app/src/locales/id/messages.po | 4 +- packages/app/src/locales/ko/messages.po | 4 +- packages/app/src/locales/pl/messages.po | 4 +- packages/app/src/locales/pt-BR/messages.po | 4 +- packages/app/src/locales/ur/messages.po | 4 +- packages/app/src/locales/zh-Hans/messages.po | 4 +- packages/app/src/locales/zh-Hant/messages.po | 4 +- .../bug3-source-mode-writeback.test.ts | 1 + .../indented-jsx-byte-sacred.test.ts | 1 + .../interior-edit-freshness.test.ts | 1 + .../keystroke-cadence-danger-space.test.ts | 1 + .../integration/producer-guard-wiring.test.ts | 1 + ...a-039-reconciliation-counters-http.test.ts | 1 + .../qa-045-restore-after-gc.test.ts | 1 + .../qa-050-rollback-undo-shipped-path.test.ts | 1 + .../integration/undo-after-rollback.test.ts | 1 + .../integration/undo-recycle-reset.test.ts | 1 + .../undo-rederive-survival.test.ts | 1 + .../app/tests/meta/playwright-cache.test.ts | 22 + packages/app/vitest.config.ts | 14 + .../src/commands/bug-report-bundle.test.ts | 104 +++- .../cli/src/commands/bug-report-bundle.ts | 84 ++- packages/cli/src/commands/mcp.ts | 1 + packages/cli/src/diagnose/bundle.test.ts | 152 +++++- packages/cli/src/diagnose/bundle.ts | 98 ++-- packages/cli/src/diagnose/path-parity.test.ts | 10 - .../cli/src/diagnose/staging-coverage.test.ts | 24 +- packages/cli/src/mcp/server.ts | 3 +- packages/cli/src/mcp/shim.test.ts | 179 ++++++- packages/cli/src/mcp/shim.ts | 31 +- packages/core/src/bridge/compose.ts | 1 + .../core/src/bridge/doc-boundary-space.ts | 1 + packages/core/src/config/schema.ts | 2 +- packages/core/src/constants/lifecycle.ts | 8 + packages/core/src/index.ts | 1 + .../core/src/markdown/comment-promoter.ts | 1 + packages/core/src/markdown/merged-walker.ts | 1 + packages/core/src/markdown/safe-url.ts | 1 + packages/core/src/server.ts | 6 + .../core/src/util/project-logs-dir.test.ts | 61 +++ packages/core/src/util/project-logs-dir.ts | 26 + .../src/util/spawn-error-log-open.test.ts | 17 +- .../core/src/util/spawn-error-log-open.ts | 4 +- packages/desktop/src/main/index.ts | 9 +- packages/desktop/src/main/window-manager.ts | 21 +- .../main/window-manager-ephemeral.test.ts | 69 ++- .../desktop/tests/main/window-manager.test.ts | 26 +- .../src/boot-project-log-retention.test.ts | 201 ++++++++ .../server/src/boot-skill-read-stall.test.ts | 2 +- packages/server/src/boot.test.ts | 2 +- packages/server/src/boot.ts | 25 +- .../server/src/collab-message-limit.test.ts | 2 +- .../server/src/console-discipline.test.ts | 2 +- packages/server/src/crash-capture.test.ts | 27 +- packages/server/src/crash-capture.ts | 2 +- packages/server/src/fs-traced.ts | 6 + .../server/src/git-preflight-boot.test.ts | 2 +- packages/server/src/index.ts | 1 + .../src/keepalive-presence-bootstrap.test.ts | 2 +- .../src/keepalive-presence-cleanup.test.ts | 2 +- .../src/lint/write-okf-schemas-wiring.test.ts | 2 +- packages/server/src/logger.test.ts | 21 +- .../src/mcp-dual-transport-rig.test-helper.ts | 2 +- .../src/mcp-lint-http-preservation.test.ts | 2 +- .../src/nested-project-authority.test.ts | 2 +- .../server/src/polled-path-watcher.test.ts | 5 +- .../server/src/project-logs-retention.test.ts | 484 ++++++++++++++++++ packages/server/src/project-logs-retention.ts | 132 +++++ packages/server/src/server-authority.test.ts | 36 ++ packages/server/src/server-authority.ts | 219 +++++--- .../server-factory.conflict-authority.test.ts | 2 +- ...-factory.conflict-ledger-lifecycle.test.ts | 2 +- packages/server/src/shutdown-progress.test.ts | 2 +- .../server/src/telemetry-file-sink.test.ts | 86 +++- packages/server/src/telemetry-file-sink.ts | 30 +- packages/server/src/telemetry.test.ts | 14 +- test-support/boot-server.test-helper.ts | 11 + .../server-authority-registry.setup.ts | 17 +- 91 files changed, 2179 insertions(+), 285 deletions(-) create mode 100644 .changeset/project-diagnostic-location.md create mode 100644 packages/app/tests/meta/playwright-cache.test.ts create mode 100644 packages/core/src/util/project-logs-dir.test.ts create mode 100644 packages/core/src/util/project-logs-dir.ts create mode 100644 packages/server/src/boot-project-log-retention.test.ts create mode 100644 packages/server/src/project-logs-retention.test.ts create mode 100644 packages/server/src/project-logs-retention.ts create mode 100644 test-support/boot-server.test-helper.ts diff --git a/.changeset/project-diagnostic-location.md b/.changeset/project-diagnostic-location.md new file mode 100644 index 000000000..572dcc896 --- /dev/null +++ b/.changeset/project-diagnostic-location.md @@ -0,0 +1,7 @@ +--- +"@inkeep/open-knowledge": patch +"@inkeep/open-knowledge-core": patch +"@inkeep/open-knowledge-server": patch +--- + +Keep server logs, diagnostic spans, and startup errors outside project folders to avoid cloud-file sync churn. Support bundles still collect diagnostics left by older versions. diff --git a/docs/content/reference/cli.mdx b/docs/content/reference/cli.mdx index c88525654..ab75cf866 100644 --- a/docs/content/reference/cli.mdx +++ b/docs/content/reference/cli.mdx @@ -199,7 +199,7 @@ The rest of the CLI exists mostly for tooling and automation. Your AI editor spa ## Remove OpenKnowledge -`ok deinit [path]` reverses `ok init` for one project (the current project when `path` is omitted): it removes the project's `.ok/` directory, OpenKnowledge's editor MCP entries, its own git-exclude lines, and shadow repo, leaving your markdown untouched. Shared editor settings and their symlinks are preserved; ignore rules for shared config files stay while those files still exist. +`ok deinit [path]` reverses `ok init` for one project (the current project when `path` is omitted): it removes the project's `.ok/` directory, OpenKnowledge's editor MCP entries, its own git-exclude lines, and shadow repo, leaving your markdown untouched. Shared editor settings and their symlinks are preserved; ignore rules for shared config files stay while those files still exist. User-level project diagnostics remain under `~/.ok/logs/projects//` (Beta: `~/.ok-beta/logs/projects//`). Later server starts with local diagnostic sinks enabled can reclaim inactive directories older than seven days; disabling the sinks does not erase existing files. `ok uninstall` removes the global footprint: running servers, credentials, the PATH shim, OpenKnowledge's editor MCP entries and bundled skills, app data, and `~/.ok`. It keeps the `~/.ok` entries every product shares: `~/.ok/machine-id`, the user skills store `~/.ok/skills`, `~/.ok/skills-lock.json`, the Global-scope skill bookkeeping in `~/.ok/local/installed-skills.json`, `~/.ok/local/skill-placements.json`, and `~/.ok/local/skill-move-retained.json`, plus the shared ownership registry `~/.ok/local/server-authority.sqlite`, its `-journal` file, and `~/.ok/local/server-authority-leases/`. These coordination files stay even with `--purge-content`. It removes only the running product's footprint: `ok-beta uninstall` removes `~/.ok-beta` and Beta's credentials and leaves Stable's `~/.ok` alone. No product's uninstall removes `~/.ok/machine-id`. With `--purge-content`, either product also removes the shared `~/.ok/skills`, unless an earlier uninstall step failed. See [Stable and Beta home folders](/docs/reference/what-open-knowledge-writes#stable-and-beta-home-folders). Your markdown and other skills in your editors' skill directories stay. An interactive run lets you select projects to deinitialize; `--yes` removes only the global footprint unless you also pass `--all-projects`, which covers current, recent, and discovered running projects. The CLI binary stays installed; the command ends by printing how to remove it (`npm uninstall -g @inkeep/open-knowledge` for the install above). diff --git a/docs/content/reference/configuration.mdx b/docs/content/reference/configuration.mdx index 79d0b5cf1..74ea51e82 100644 --- a/docs/content/reference/configuration.mdx +++ b/docs/content/reference/configuration.mdx @@ -73,7 +73,7 @@ A key set in a file more specific than its scope (a user-scope key in `.ok/confi | `slides.enabled` | boolean | `false` | user | Whether the [Slidev plugin](/docs/plugins/slidev) appears in **Settings → Plugins**. Off by default. When on, a document whose frontmatter contains `slides: true` gets an **Open in Slidev** action that opens the deck in its own window, rendered by a [Slidev](https://sli.dev/) you install yourself — OpenKnowledge does not download or bundle one. Install it globally (`npm install -g @slidev/cli @slidev/theme-default`) or add those packages to the project (a project-local install takes precedence). Slidev themes ship separately from the CLI and a deck cannot open without the theme it declares. Desktop only; the action stays hidden until a `slidev` binary resolves. | | `agents.autoApproveOkTools` | boolean | `true` | user | Auto-approve OpenKnowledge's own MCP tools for agents launched from the built-in [docked terminal](/docs/features/editor#terminal) (plus the `ok open` command on Claude; `ok-beta open` for Beta) and for in-app Agents-panel chats with any ACP agent, so reading and writing the knowledge base runs without a per-call approval prompt. Destructive tools (`delete`, `move`, `share_link`, `install`, `import`) still prompt; other shell commands and non-OpenKnowledge file edits are untouched. Per-machine preference; toggle from **Settings → Terminal**. Best-effort per agent — Claude uses an allow/ask list, Codex uses its per-server `approve` mode (and only when its OpenKnowledge entry is already configured). | | `agents.browserTools` | boolean | `false` | user | Give in-app Claude Code and Codex chats a browser they can drive: each one started after you turn it on gets the [Playwright MCP](https://github.com/microsoft/playwright-mcp) server, so the agent can open pages, click, type, and take screenshots. It is launched with the `npx` on the agent's PATH from OpenKnowledge's own folder, so npm never uses the project's npm files; without a usable `npx`, or when the browser can't be set up, the chat starts without it and says so in the conversation. Needs Google Chrome and Node.js installed. The browser opens in its own window (headless on Linux with no display) with a fresh in-memory profile; everything the browser saves goes to a folder for the chat under `~/.ok/agent-browser/` (Beta: `~/.ok-beta/agent-browser/`) and is removed with the chat. Every browser action asks for approval, with no option to always allow it, unless the agent's own mode or permission settings skip approvals. Other agents' chats don't get it. Per-machine preference; toggle from **Settings → Preferences**. | -| `telemetry.localSink.enabled` | boolean | `true` | project | Write local diagnostic spans + logs under `.ok/local/` for `ok diagnose bundle` to collect. Local-only: nothing leaves the machine until you run `bundle`. Set `false` for sensitive workspaces. | +| `telemetry.localSink.enabled` | boolean | `true` | project | Write per-project diagnostic spans + logs under `~/.ok/logs/projects//` (Beta: `~/.ok-beta/logs/projects//`) for `ok diagnose bundle` to collect. Files stay local unless shared, including in a support bundle. Set `false` to stop normal span/log writes and their startup cleanup; existing files remain. Fatal-crash records and detached-server startup stderr are separate and remain enabled. | | `telemetry.localSink.spans.maxBytes` | number | `52428800` (~50 MB) | project | Max size of the local diagnostic spans file before it rotates. | | `telemetry.localSink.logs.maxBytes` | number | `26214400` (~25 MB) | project | Max size of the local diagnostic logs file before it rotates. | | `telemetry.localSink.attributeDenylist` | `string[]` | 8 credential keys | project | Attribute keys whose values are redacted (`[REDACTED]`) before any local span/log is written. Extends the built-in denylist (`authorization`, `password`, `cookie`, etc.). | @@ -143,7 +143,7 @@ Most users never set these — the schema settings above cover the common cases. | --- | --- | | `OPEN_KNOWLEDGE_GITHUB_CLIENT_ID` | Override the GitHub OAuth App client ID | | `OK_SHOWALL_MAX_ENTRIES` | Max entries the sidebar's disk-walk listing returns per level before it stops and marks the list truncated (default `50000`) | -| `OK_BRIDGE_TOLERANCE_TELEMETRY` | Set to `1` to record each bridge-tolerance-class fire as a JSONL line in `/.ok/local/tolerance-telemetry.jsonl` (opt-in, local-only, off by default; unstable diagnostic for triaging markdown-fidelity incidents). **Records doc paths in cleartext** — independent of `telemetry.localSink`'s `attributeDenylist`, which redacts span/log attributes, not this file. Deliberately outside the `.ok/local/telemetry/` subtree `ok diagnose bundle` harvests, so the unredacted paths never ship in a bug bundle. | +| `OK_BRIDGE_TOLERANCE_TELEMETRY` | Set to `1` to record each bridge-tolerance-class fire as a JSONL line in `/.ok/local/tolerance-telemetry.jsonl` (opt-in, local-only, off by default; unstable diagnostic for triaging markdown-fidelity incidents). **Records doc paths in cleartext** — independent of `telemetry.localSink`'s `attributeDenylist`, which redacts span/log attributes, not this file. This file is not collected by `ok diagnose bundle`, so its unredacted paths never ship in a bug bundle. | | `OTEL_SDK_DISABLED` | OpenTelemetry OTLP push gate. Inverted sense: set to `false` to **enable** push (it is off by default). | | `OK_BUG_REPORT_INTAKE_URL` | Base URL of the intake the desktop app uploads to: reports sent from **Help → Report a bug…**, and images attached to the feedback form. Defaults to `https://openknowledge.ai`. Must be `https:` — plain `http:` is accepted only for loopback hosts (local testing). | | `OK_FEEDBACK_INTAKE_ORIGIN` | Origin the optional [uninstall feedback](/docs/reference/what-open-knowledge-writes#what-leaves-your-machine) submission posts to (default `https://openknowledge.ai`). Must be `https:` — plain `http:` is accepted only for loopback hosts (local testing). An unusable value drops the submission rather than falling back to the default. | diff --git a/docs/content/reference/what-open-knowledge-writes.mdx b/docs/content/reference/what-open-knowledge-writes.mdx index 2a9069fc9..4c453e9e8 100644 --- a/docs/content/reference/what-open-knowledge-writes.mdx +++ b/docs/content/reference/what-open-knowledge-writes.mdx @@ -144,7 +144,7 @@ An earlier Beta stored its GitHub token in the OS credential store under the ser ### While a project is open -The app runs a collaboration server for the open project. It writes runtime state, gitignored under `.ok/local/`, plus a shadow git repo that powers the timeline and recovery features: +The app runs a collaboration server for the open project. It keeps coordination state gitignored under `.ok/local/`, diagnostics in the product's user-level `logs/projects//`, and a shadow git repo that powers the timeline and recovery features: | Path | What it is | Scope | | --- | --- | --- | @@ -155,13 +155,14 @@ The app runs a collaboration server for the open project. It writes runtime stat | `.ok/local/stale-external-writes.json` | Recovery snapshot for a rejected stale external save. It can hold the acknowledged, rejected, and retained document text. Protection expires after 30 minutes, but an unresolved snapshot stays until you resolve the conflict | In-project, gitignored | | `.ok/local/doc-notices.json` | The notice a document or settings file shows after both versions of an edit were kept, or a version was saved to its history or the backup folder instead of merged, and, for a document or skill but not a settings file, a separate notice while a save of it is held, removed once that save lands or OpenKnowledge restarts with nothing held: document names, file paths, times, causes, and history entry ids, never document text. A notice is removed when you dismiss it, when its document is deleted, when the version it names leaves the timeline, or when more than 200 newer notices, counted across all branches, push it out. It follows its document through a rename, and each branch keeps its own. If the app cannot read the file, it starts with no notices | In-project, gitignored | | `.ok/local/receipts.jsonl` | One record per document OpenKnowledge has opened or saved: a hash of the version it and the file last agreed on, its size, and the file's identity and modification time. No document text. OpenKnowledge keeps the same record in memory while it runs, and a save of a Markdown, Mermaid, Excalidraw or plain-text document compares the file with that record first, so an outside edit is merged, or saved to history, instead of written over; settings files compare against the last version OpenKnowledge read or wrote instead, and other writers do not check it yet. After a restart the file is only used to count documents changed while OpenKnowledge was closed. Safe to delete: the next load rebuilds it | In-project, gitignored | -| `.ok/local/state.json`, `.ok/local/last-spawn-error.log` | State-schema manifest and the last detached-spawn error log | In-project, gitignored | +| `.ok/local/state.json` | State-schema manifest | In-project, gitignored | +| `~/.ok/logs/projects//last-spawn-error.log` | Startup stderr from detached servers; starts over on the next attempt after reaching 256 KB | Outside-project (home dir) | | `.ok/local/acp-agents.json` | Read, not written: your own list of [custom in-app agents](../features/editor.mdx#add-your-own-in-app-agent), re-read every time the agent catalog is built | In-project, gitignored | | `.ok/local/acp-launch-failures.log` | In-app agent launches that failed: when, which agent, why, and the last lines the agent printed before it exited, so a bug report shows the cause without a re-run. Starts over once it passes 256 KB | In-project, gitignored | | `.ok/local/last-server-exit.json` | Why the collaboration server the app started last exited — a timestamp, its process id, the exit code, the signal that ended it, and which part of the app was watching. One record, overwritten on each exit; a server the app only attached to, or one you started yourself with `ok start`, leaves none. No document content or paths | In-project, gitignored | | `.ok/local/last-server-crash.json` | The collaboration server's own account of a crash it did not survive — a timestamp, the error message and stack, its process id, and how long it had been running. Written by the server itself, so it covers `ok start` too. Holds the most recent crash only | In-project, gitignored | -| `.ok/local/telemetry/spans-*.jsonl` | Local diagnostic spans, rotated at ~50 MB | In-project, gitignored | -| `.ok/local/logs/server-*.jsonl` | Local server logs, rotated at ~25 MB | In-project, gitignored | +| `~/.ok/logs/projects//spans-*.jsonl` | Local diagnostic spans, rotated at ~50 MB | Outside-project (home dir) | +| `~/.ok/logs/projects//server-*.jsonl` | Local server logs, rotated at ~25 MB | Outside-project (home dir) | | `.ok/local/cache//backlinks.json` | Backlink-graph cache | In-project, gitignored | | `.ok/local/installed-skills.json` | Per-project skill-install markers (which editors each skill is projected into) | In-project, gitignored | | `.ok/local/agent-skill/open-knowledge/` | Runtime copy of the OpenKnowledge project skill (`SKILL.md` and `references/`), refreshed before in-app agent processes start. New in-app agent sessions receive this guidance automatically, even when the skill is not installed into an editor | In-project, gitignored with the `local/` ignore rule | @@ -171,7 +172,11 @@ The app runs a collaboration server for the open project. It writes runtime stat | `.ok/worktrees//` | Git worktree checkout created when you open a branch in its own window (project switcher); the path is also added to `.git/info/exclude` | In-project, gitignored | | `.git/ok/` — or `.git/worktrees//ok/` in a linked worktree, or the enclosing repo's `.git/ok-/` when your project is a subfolder of a larger repo | Shadow git repo holding per-writer work-in-progress refs, plus the versions OpenKnowledge set aside when an outside change could not be merged: of documents, Mermaid, Excalidraw and plain-text files, global and external skills, and the project, per-machine (`.ok/local/config.yml`) and user (`~/.ok/global.yml`) settings files. Those are saved as their own history entries, which `info/exclude` does not filter, so a copy of this folder can carry per-machine settings, your user settings and global skill text; each kind keeps its 50 newest per document, with no time limit. Always inside a git admin dir, never in your content tree. Its own `info/exclude` file, written when OpenKnowledge opens the repo, keeps machine-local runtime state — `.ok/local/` and `.ok/worktrees/`, plus the pre-`.ok/local/` equivalents that older projects still keep at the `.ok/` root (`.ok/principal.json`, `.ok/server.lock`, `.ok/cache/`, …) — the `.tmp.` files an in-flight atomic write leaves behind, and your project's `.git` when it resolves to a directory inside the project under another name (a `.git.nosync` symlink or a separate git dir), out of every work-in-progress version, the way `.ok/.gitignore` keeps them out of git | In-project, gitignored (inside `.git/`) | -Diagnostic logs and telemetry are **on by default but local-only** — credential-bearing attributes are redacted (`[REDACTED]`) before anything is written, the files rotate at the size caps above, and nothing leaves the machine until you explicitly run `ok diagnose bundle`. Turn the local sink off with `telemetry.localSink.enabled: false`; see [Configuration](./configuration.mdx). +The project hash comes from the canonical project-folder path, so projects with the same name keep separate logs. Existing project-local diagnostic files are left in place; support bundles still collect them, using `legacy/` entries when both old and current files exist. + +When local diagnostic sinks are enabled, server startup removes inactive project log directories only when the directory and all recognized diagnostic files are older than seven days. With `telemetry.localSink.enabled: false` or `OK_DISABLE_LOCAL_SINK=1`, startup skips this cleanup; detached-server launch only refreshes its own stderr directory. Live servers remain protected even when quiet; uncertain ownership, symlinks, or unfamiliar contents prevent cleanup. This runs on later launches, not on a timer. `ok deinit` leaves these user-level diagnostics for that cleanup rather than deleting them immediately. + +Diagnostic logs and telemetry are **on by default but local-only** — credential-bearing attributes are redacted (`[REDACTED]`) before anything is written, the files rotate at the size caps above, and the files stay local unless shared, including in a support bundle. Turn the local sink off with `telemetry.localSink.enabled: false`; see [Configuration](./configuration.mdx). On each launch and project open, the app also runs **repair sweeps**: it rewrites *existing* `open-knowledge` MCP entries (in the editor configs above) to the current canonical form, and removes any stale `open-knowledge-ui` entry left in `.claude/launch.json` by an older OK version. It **never adds** an MCP entry to an editor that doesn't already have one. Same `OK_RECLAIM_DISABLE=1` opt-out. @@ -205,7 +210,7 @@ Beta writes the `~/.ok/` entries below under `~/.ok-beta/` instead, except the s | `~/.ok/themes/` | Your saved color themes from the [Themes plugin](../plugins/themes.mdx), one Tinted Theming base16 YAML file per theme, written when you create or edit a theme | No | | `~/.ok/edit-backups/discarded/projects///`, `~/.ok/edit-backups/discarded//` | A copy of a version OpenKnowledge set aside but could not save to the project's history, for example in a project with no shadow repo: the outside version of a Mermaid, Excalidraw or plain-text file, a skill's outside version that could not be merged, or a settings version that was replaced. Files of a project go in a folder for that project (`` is a short hash of its content folder), so two projects with the same file keep separate copies; global and external skills keep one folder each. Each copy is named for the time it was kept, and the five newest per folder stay. The notice on the document, or in Settings, names the file | No | | `~/.ok/mcp-status.json` | First-launch MCP consent record | No | -| `~/.ok/logs/`, `~/.ok/bug-reports/` | Desktop and CLI logs, and bug-report bundles from `ok bug-report` and the app's **Help → Report a bug…** dialog (`ok diagnose bundle` writes into the project, under `.ok/local/diagnostics/`). Each report from the dialog also gets a small YAML record holding its send state and a secret-redacted copy of the note you wrote, so report history can still show what a report was about. The zip is deleted once the report is sent; the record is kept, and both go when you delete the report from report history. If that record ever can't be read back, OpenKnowledge leaves it exactly as it is rather than overwriting your note, and writes a second tiny `.sent.yaml` file beside it when a report does send, so history still shows it as sent instead of offering to send it again | No | +| `~/.ok/logs/` (including `projects//`), `~/.ok/bug-reports/` | Desktop and CLI logs, and bug-report bundles from `ok bug-report` and the app's **Help → Report a bug…** dialog (`ok diagnose bundle` writes into the project, under `.ok/local/diagnostics/`). Each report from the dialog also gets a small YAML record holding its send state and a secret-redacted copy of the note you wrote, so report history can still show what a report was about. The zip is deleted once the report is sent; the record is kept, and both go when you delete the report from report history. If that record ever can't be read back, OpenKnowledge leaves it exactly as it is rather than overwriting your note, and writes a second tiny `.sent.yaml` file beside it when a report does send, so history still shows it as sent instead of offering to send it again | No | | `~/.ok/bin/`, `~/.ok/env.sh` | `PATH` shim (see [Shell and `PATH`](#shell-and-path) above) | No | **Auth tokens** (for GitHub sync, sharing, and cloning) are stored in your OS credential store under the service name `open-knowledge` (`open-knowledge-beta` for Beta) — the **Keychain** on macOS, **Credential Manager** on Windows, and the Secret Service keyring (libsecret) on Linux. The `~/.ok/auth.yml` file is only a fallback for headless environments where the credential store isn't reachable; when it becomes available, the token is migrated into it and the file copy is removed. @@ -300,12 +305,13 @@ On Windows, Claude Desktop and OpenCode configs live under `%APPDATA%\Claude\` a | `.ok/local/stale-external-writes.json` | Recovery snapshot for a rejected stale external save. It can hold the acknowledged, rejected, and retained document text. Protection expires after 30 minutes, but an unresolved snapshot stays until you resolve the conflict | In-project, gitignored | | `.ok/local/doc-notices.json` | The notice a document or settings file shows after both versions of an edit were kept, or a version was saved to its history or the backup folder instead of merged, and, for a document or skill but not a settings file, a separate notice while a save of it is held, removed once that save lands or OpenKnowledge restarts with nothing held: document names, file paths, times, causes, and history entry ids, never document text. A notice is removed when you dismiss it, when its document is deleted, when the version it names leaves the timeline, or when more than 200 newer notices, counted across all branches, push it out. It follows its document through a rename, and each branch keeps its own. If the app cannot read the file, it starts with no notices | In-project, gitignored | | `.ok/local/receipts.jsonl` | One record per document OpenKnowledge has opened or saved: a hash of the version it and the file last agreed on, its size, and the file's identity and modification time. No document text. OpenKnowledge keeps the same record in memory while it runs, and a save of a Markdown, Mermaid, Excalidraw or plain-text document compares the file with that record first, so an outside edit is merged, or saved to history, instead of written over; settings files compare against the last version OpenKnowledge read or wrote instead, and other writers do not check it yet. After a restart the file is only used to count documents changed while OpenKnowledge was closed. Safe to delete: the next load rebuilds it | In-project, gitignored | -| `.ok/local/state.json`, `.ok/local/last-spawn-error.log` | State-schema manifest and the last detached-spawn error log | In-project, gitignored | +| `.ok/local/state.json` | State-schema manifest | In-project, gitignored | +| `~/.ok/logs/projects//last-spawn-error.log` | Startup stderr from detached servers; starts over on the next attempt after reaching 256 KB | Outside-project (home dir) | | `.ok/local/acp-agents.json` | Read, not written: your own list of [custom in-app agents](../features/editor.mdx#add-your-own-in-app-agent), re-read every time the agent catalog is built | In-project, gitignored | | `.ok/local/acp-launch-failures.log` | In-app agent launches that failed: when, which agent, why, and the last lines the agent printed before it exited, so a bug report shows the cause without a re-run. Starts over once it passes 256 KB | In-project, gitignored | | `.ok/local/last-server-crash.json` | The server's own account of a crash it did not survive — a timestamp, the error message and stack, its process id, and how long it had been running. Holds the most recent crash only | In-project, gitignored | -| `.ok/local/telemetry/spans-*.jsonl` | Local diagnostic spans, rotated at ~50 MB | In-project, gitignored | -| `.ok/local/logs/server-*.jsonl` | Local server logs, rotated at ~25 MB | In-project, gitignored | +| `~/.ok/logs/projects//spans-*.jsonl` | Local diagnostic spans, rotated at ~50 MB | Outside-project (home dir) | +| `~/.ok/logs/projects//server-*.jsonl` | Local server logs, rotated at ~25 MB | Outside-project (home dir) | | `.ok/local/cache//backlinks.json` | Backlink-graph cache | In-project, gitignored | | `.ok/local/installed-skills.json` | Per-project skill-install markers (which editors each skill is projected into) | In-project, gitignored | | `.ok/local/agent-skill/open-knowledge/` | Runtime copy of the OpenKnowledge project skill (`SKILL.md` and `references/`), refreshed before in-app agent processes start. New in-app agent sessions receive this guidance automatically, even when the skill is not installed into an editor | In-project, gitignored with the `local/` ignore rule | @@ -313,7 +319,11 @@ On Windows, Claude Desktop and OpenCode configs live under `%APPDATA%\Claude\` a | `.ok/local/comments/` | [Comments](/docs/features/comments) left on documents, one JSON file each — the note, the passage it quotes, and whether it has been sent | In-project, gitignored | | `.git/ok/` — or `.git/worktrees//ok/` in a linked worktree, or the enclosing repo's `.git/ok-/` when your project is a subfolder of a larger repo | Shadow git repo holding per-writer work-in-progress refs (powers the timeline), plus the versions OpenKnowledge set aside when an outside change could not be merged: of documents, Mermaid, Excalidraw and plain-text files, global and external skills, and the project, per-machine (`.ok/local/config.yml`) and user (`~/.ok/global.yml`) settings files. Those are saved as their own history entries, which `info/exclude` does not filter, so a copy of this folder can carry per-machine settings, your user settings and global skill text; each kind keeps its 50 newest per document, with no time limit. Always inside a git admin dir, never in your content tree. Its own `info/exclude` file, written when OpenKnowledge opens the repo, keeps machine-local runtime state — `.ok/local/` and `.ok/worktrees/`, plus the pre-`.ok/local/` equivalents that older projects still keep at the `.ok/` root (`.ok/principal.json`, `.ok/server.lock`, `.ok/cache/`, …) — the `.tmp.` files an in-flight atomic write leaves behind, and your project's `.git` when it resolves to a directory inside the project under another name (a `.git.nosync` symlink or a separate git dir), out of every work-in-progress version, the way `.ok/.gitignore` keeps them out of git | In-project, gitignored (inside `.git/`) | -Diagnostic logs and telemetry are **on by default but local-only** — credential-bearing attributes are redacted (`[REDACTED]`) before anything is written, the files rotate at the size caps above, and nothing leaves the machine until you explicitly run `ok diagnose bundle`. Turn the local sink off with `telemetry.localSink.enabled: false`; see [Configuration](./configuration.mdx). +The project hash comes from the canonical project-folder path, so projects with the same name keep separate logs. Existing project-local diagnostic files are left in place; support bundles still collect them, using `legacy/` entries when both old and current files exist. + +When local diagnostic sinks are enabled, server startup removes inactive project log directories only when the directory and all recognized diagnostic files are older than seven days. With `telemetry.localSink.enabled: false` or `OK_DISABLE_LOCAL_SINK=1`, startup skips this cleanup; detached-server launch only refreshes its own stderr directory. Live servers remain protected even when quiet; uncertain ownership, symlinks, or unfamiliar contents prevent cleanup. This runs on later launches, not on a timer. `ok deinit` leaves these user-level diagnostics for that cleanup rather than deleting them immediately. + +Diagnostic logs and telemetry are **on by default but local-only** — credential-bearing attributes are redacted (`[REDACTED]`) before anything is written, the files rotate at the size caps above, and the files stay local unless shared, including in a support bundle. Turn the local sink off with `telemetry.localSink.enabled: false`; see [Configuration](./configuration.mdx). On each boot, `ok start` also runs **repair sweeps**: it re-checks a few things that drift as OpenKnowledge updates and rewrites any that have fallen out of the current canonical form — a no-op when nothing has changed, and it **never adds** an entry that isn't already there. @@ -328,7 +338,7 @@ Projects initialized by an older CLI may lack the `local/` rule in `.ok/.gitigno ### `ok mcp` -`ok mcp` is the stdio bridge your editor spawns to talk to the server. It **writes nothing to disk** — it routes tool calls to a running `ok start` backend (starting one on the first tool call that needs it — reads included). +`ok mcp` is the stdio bridge your editor spawns to talk to the server. It routes tool calls to a running `ok start` backend (starting one on the first tool call that needs it — reads included). When it starts a backend, it captures startup stderr in the user-level project logs listed above. ### Home directory and credentials (`~/.ok/`) @@ -347,7 +357,7 @@ Beta writes the `~/.ok/` entries below under `~/.ok-beta/` instead, except the s | `_npx//` in the npm cache that launch uses: your own npm cache (`~/.npm` unless your `npm_config_cache` says otherwise) when the agent runs through your system npx, or OpenKnowledge's managed Node.js runtime cache at `~/.ok/runtimes/cache/node` when no usable system Node.js was found and the agent runs on that runtime | Written when OpenKnowledge installs or launches an in-app agent through npx, a custom command such as `npx -y ` included, and when the browser server a Claude Code or Codex chat starts while **Let agent chats use a browser** is on installs `@playwright/mcp`. OpenKnowledge repairs the entry only for catalog-managed agents and the browser server, never for custom agent commands or uv-managed adapters. When npm reports that the installed copy is missing as a catalog-managed agent starts, OpenKnowledge installs it again once. When npm reports that the entry is missing its `package.json`, OpenKnowledge deletes that one directory and relaunches the agent once, which installs it again. The browser server's entry is cleared the same way and reinstalled once | No | | `~/.ok/themes/` | Your saved color themes from the [Themes plugin](../plugins/themes.mdx), one Tinted Theming base16 YAML file per theme, written when you create or edit a theme | No | | `~/.ok/edit-backups/discarded/projects///`, `~/.ok/edit-backups/discarded//` | A copy of a version OpenKnowledge set aside but could not save to the project's history, for example in a project with no shadow repo: the outside version of a Mermaid, Excalidraw or plain-text file, a skill's outside version that could not be merged, or a settings version that was replaced. Files of a project go in a folder for that project (`` is a short hash of its content folder), so two projects with the same file keep separate copies; global and external skills keep one folder each. Each copy is named for the time it was kept, and the five newest per folder stay. The notice on the document, or in Settings, names the file | No | -| `~/.ok/logs/` | CLI logs (`cli..log`), size- and age-capped | No | +| `~/.ok/logs/` | CLI logs (`cli..log`), size- and age-capped, plus per-project server logs, spans, and startup stderr under `projects//` | No | | `~/.ok/bug-reports/` | Bug-report bundles from `ok bug-report` — the desktop app's **Help → Report a bug…** dialog writes here too (`ok diagnose bundle` writes into the project, under `.ok/local/diagnostics/`). A report filed from the dialog also leaves a small YAML record beside its bundle, holding the report's send state and a secret-redacted copy of the note you wrote so report history can name it. That record outlives the bundle: the zip is deleted once the report is sent, and the record is kept until you delete the report from report history. If the record ever can't be read back, OpenKnowledge leaves it exactly as it is rather than overwriting your note, and writes a second tiny `.sent.yaml` file beside it when a report does send. That marker holds only the send time and the report's reference, never your note, and it is removed with the rest of the report | No | **Auth tokens** (for GitHub sync, sharing, and cloning) are stored in your operating system's secure credential store when one is available — the macOS **Keychain** (service `open-knowledge`, or `open-knowledge-beta` for Beta), or the platform equivalent (Windows Credential Manager, Linux Secret Service) — falling back to a `0600` `~/.ok/auth.yml` file on headless systems (a Linux server, CI) where none is reachable. diff --git a/knip.config.ts b/knip.config.ts index e04bf5228..fac2580fe 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -157,11 +157,15 @@ export default { 'src/shadow-fixture-launch.test-helper.mjs', ], project: 'src/**', - ignoreDependencies: ['@types/shell-quote'], + ignoreDependencies: [ + '@types/shell-quote', + ], }, 'packages/cli': { entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'], - ignoreDependencies: ['yjs'], + ignoreDependencies: [ + 'yjs', + ], }, 'packages/desktop': { entry: [ diff --git a/packages/app/src/components/ConnectingBanner.dom.test.tsx b/packages/app/src/components/ConnectingBanner.dom.test.tsx index da609afbd..7ef2f56e2 100644 --- a/packages/app/src/components/ConnectingBanner.dom.test.tsx +++ b/packages/app/src/components/ConnectingBanner.dom.test.tsx @@ -75,12 +75,15 @@ describe('ConnectingBanner terminal copy names the running channel CLI', () => { test.each([ [undefined, 'ok'], ['ok-beta', 'ok-beta'], - ])('an unreachable server names %s status and start', (cliCommandName, command) => { - (window as unknown as { okDesktop?: unknown }).okDesktop = { cliCommandName }; - collabLastError = { kind: 'error', code: 'network' }; - render(); - expect(screen.getByRole('alert').textContent).toContain( - `Couldn't reach collab server — network error (is \`${command} start\` running?). Try ${command} status or check .ok/local/last-spawn-error.log.`, - ); - }); + ])( + 'an unreachable server offers %s status and diagnostic collection', + (cliCommandName, command) => { + (window as unknown as { okDesktop?: unknown }).okDesktop = { cliCommandName }; + collabLastError = { kind: 'error', code: 'network' }; + render(); + expect(screen.getByRole('alert').textContent).toContain( + `Couldn't reach collab server — network error (is \`${command} start\` running?). Try ${command} status or collect logs with ${command} diagnose bundle.`, + ); + }, + ); }); diff --git a/packages/app/src/components/ConnectingBanner.tsx b/packages/app/src/components/ConnectingBanner.tsx index 8575811e6..9164033da 100644 --- a/packages/app/src/components/ConnectingBanner.tsx +++ b/packages/app/src/components/ConnectingBanner.tsx @@ -81,8 +81,9 @@ export function ConnectingBanner() { ) : ( Couldn't reach collab server — {errorDetail}. Try{' '} - {cliCommand} status or check{' '} - .ok/local/last-spawn-error.log. + {cliCommand} status or collect + logs with{' '} + {cliCommand} diagnose bundle. )} diff --git a/packages/app/src/locales/ar/messages.po b/packages/app/src/locales/ar/messages.po index 5a6023ba4..23c25a245 100644 --- a/packages/app/src/locales/ar/messages.po +++ b/packages/app/src/locales/ar/messages.po @@ -3115,8 +3115,8 @@ msgstr "تعذّر الوصول إلى {displayName}. يُرجى المحاول msgid "Couldn't reach {displayName} — try again?" msgstr "تعذّر الوصول إلى {displayName}. هل تريد المحاولة مرة أخرى؟" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "تعذّر الوصول إلى خادم التعاون — {errorDetail}. جرّب <0>{cliCommand} status أو راجع <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "تعذّر الوصول إلى خادم التعاون — {errorDetail}. جرّب <0>{cliCommand} status أو اجمع السجلات باستخدام <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "تعذّر الوصول إلى GitHub. هل تعيد المحاولة؟" diff --git a/packages/app/src/locales/bn/messages.po b/packages/app/src/locales/bn/messages.po index c1f11ea00..6c154d6b1 100644 --- a/packages/app/src/locales/bn/messages.po +++ b/packages/app/src/locales/bn/messages.po @@ -3111,8 +3111,8 @@ msgstr "{displayName}-এ পৌঁছানো যায়নি। অনু msgid "Couldn't reach {displayName} — try again?" msgstr "{displayName}-এ পৌঁছানো যায়নি। আবার চেষ্টা করবেন?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "সহযোগিতা সার্ভারের নাগাল পাওয়া যায়নি — {errorDetail}। <0>{cliCommand} status চেষ্টা করুন বা <1>.ok/local/last-spawn-error.log দেখুন।" +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "সহযোগিতা সার্ভারের নাগাল পাওয়া যায়নি — {errorDetail}। <0>{cliCommand} status চেষ্টা করুন অথবা <1>{cliCommand} diagnose bundle দিয়ে লগ সংগ্রহ করুন।" msgid "Couldn't reach GitHub. Try again?" msgstr "GitHub-এর নাগাল পাওয়া যায়নি। আবার চেষ্টা করবেন?" diff --git a/packages/app/src/locales/en/messages.po b/packages/app/src/locales/en/messages.po index befd35f4f..0c9137697 100644 --- a/packages/app/src/locales/en/messages.po +++ b/packages/app/src/locales/en/messages.po @@ -3115,8 +3115,8 @@ msgstr "Couldn't reach {displayName} — please try again later." msgid "Couldn't reach {displayName} — try again?" msgstr "Couldn't reach {displayName} — try again?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "Couldn't reach GitHub. Try again?" diff --git a/packages/app/src/locales/es/messages.po b/packages/app/src/locales/es/messages.po index 71c8105e8..3f70631a2 100644 --- a/packages/app/src/locales/es/messages.po +++ b/packages/app/src/locales/es/messages.po @@ -3115,8 +3115,8 @@ msgstr "No se pudo conectar con {displayName}. Inténtalo de nuevo más tarde." msgid "Couldn't reach {displayName} — try again?" msgstr "No se pudo conectar con {displayName}. ¿Reintentar?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "No se pudo contactar con el servidor de colaboración: {errorDetail}. Prueba <0>{cliCommand} status o revisa <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "No se pudo contactar con el servidor de colaboración: {errorDetail}. Prueba <0>{cliCommand} status o recopila los registros con <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "No se pudo contactar con GitHub. ¿Reintentar?" diff --git a/packages/app/src/locales/fr/messages.po b/packages/app/src/locales/fr/messages.po index d1f07d8c5..7bb586121 100644 --- a/packages/app/src/locales/fr/messages.po +++ b/packages/app/src/locales/fr/messages.po @@ -3115,8 +3115,8 @@ msgstr "Impossible de joindre {displayName}. Veuillez réessayer plus tard." msgid "Couldn't reach {displayName} — try again?" msgstr "Impossible de joindre {displayName}. Réessayer ?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "Impossible de joindre le serveur de collaboration — {errorDetail}. Essayez <0>{cliCommand} status ou consultez <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "Impossible de joindre le serveur de collaboration — {errorDetail}. Essayez <0>{cliCommand} status ou collectez les journaux avec <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "Impossible de joindre GitHub. Réessayer ?" diff --git a/packages/app/src/locales/hi/messages.po b/packages/app/src/locales/hi/messages.po index eaa903ca5..2db275589 100644 --- a/packages/app/src/locales/hi/messages.po +++ b/packages/app/src/locales/hi/messages.po @@ -3111,8 +3111,8 @@ msgstr "{displayName} तक नहीं पहुँचा जा सका। msgid "Couldn't reach {displayName} — try again?" msgstr "{displayName} तक नहीं पहुँचा जा सका। फिर कोशिश करें?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "सहयोग सर्वर तक नहीं पहुँचा जा सका — {errorDetail}. <0>{cliCommand} status आज़माएँ या <1>.ok/local/last-spawn-error.log देखें।" +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "सहयोग सर्वर तक नहीं पहुँचा जा सका — {errorDetail}. <0>{cliCommand} status आज़माएँ या <1>{cliCommand} diagnose bundle से लॉग इकट्ठा करें।" msgid "Couldn't reach GitHub. Try again?" msgstr "GitHub तक नहीं पहुँचा जा सका। फिर कोशिश करें?" diff --git a/packages/app/src/locales/id/messages.po b/packages/app/src/locales/id/messages.po index b9b8dcbd9..c2d18aa9a 100644 --- a/packages/app/src/locales/id/messages.po +++ b/packages/app/src/locales/id/messages.po @@ -3111,8 +3111,8 @@ msgstr "Tidak dapat menghubungi {displayName}. Silakan coba lagi nanti." msgid "Couldn't reach {displayName} — try again?" msgstr "Tidak dapat menghubungi {displayName}. Coba lagi?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "Server kolaborasi tidak bisa dihubungi — {errorDetail}. Coba <0>{cliCommand} status atau periksa <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "Server kolaborasi tidak bisa dihubungi — {errorDetail}. Coba <0>{cliCommand} status atau kumpulkan log dengan <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "GitHub tidak bisa dihubungi. Coba lagi?" diff --git a/packages/app/src/locales/ko/messages.po b/packages/app/src/locales/ko/messages.po index 7ba54715c..91049c58a 100644 --- a/packages/app/src/locales/ko/messages.po +++ b/packages/app/src/locales/ko/messages.po @@ -3115,8 +3115,8 @@ msgstr "{displayName} 연결 실패. 잠시 후 다시 시도하세요." msgid "Couldn't reach {displayName} — try again?" msgstr "{displayName} 연결 실패. 다시 시도할까요?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "협업 서버 연결 실패: {errorDetail}. <0>{cliCommand} status 실행 또는 <1>.ok/local/last-spawn-error.log 확인." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "협업 서버 연결 실패: {errorDetail}. <0>{cliCommand} status를 시도하거나 <1>{cliCommand} diagnose bundle로 로그를 수집하세요." msgid "Couldn't reach GitHub. Try again?" msgstr "GitHub에 연결할 수 없습니다. 다시 시도할까요?" diff --git a/packages/app/src/locales/pl/messages.po b/packages/app/src/locales/pl/messages.po index 90e6941df..a94888947 100644 --- a/packages/app/src/locales/pl/messages.po +++ b/packages/app/src/locales/pl/messages.po @@ -3115,8 +3115,8 @@ msgstr "Nie udało się połączyć z {displayName} — spróbuj ponownie późn msgid "Couldn't reach {displayName} — try again?" msgstr "Nie udało się połączyć z {displayName} — spróbować ponownie?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "Nie udało się połączyć z serwerem współpracy — {errorDetail}. Spróbuj <0>{cliCommand} status lub sprawdź <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "Nie udało się połączyć z serwerem współpracy — {errorDetail}. Spróbuj <0>{cliCommand} status lub zbierz logi poleceniem <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "Nie udało się połączyć z serwisem GitHub. Spróbować ponownie?" diff --git a/packages/app/src/locales/pt-BR/messages.po b/packages/app/src/locales/pt-BR/messages.po index c7e3c1196..fc99c30a5 100644 --- a/packages/app/src/locales/pt-BR/messages.po +++ b/packages/app/src/locales/pt-BR/messages.po @@ -3111,8 +3111,8 @@ msgstr "Não foi possível acessar {displayName}. Tente novamente mais tarde." msgid "Couldn't reach {displayName} — try again?" msgstr "Não foi possível acessar {displayName}. Tentar de novo?" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "Não deu para contatar o servidor de colaboração — {errorDetail}. Tente <0>{cliCommand} status ou verifique <1>.ok/local/last-spawn-error.log." +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "Não deu para contatar o servidor de colaboração — {errorDetail}. Tente <0>{cliCommand} status ou colete os logs com <1>{cliCommand} diagnose bundle." msgid "Couldn't reach GitHub. Try again?" msgstr "Não deu para contatar o GitHub. Tentar de novo?" diff --git a/packages/app/src/locales/ur/messages.po b/packages/app/src/locales/ur/messages.po index 49918906d..7b9be9a6e 100644 --- a/packages/app/src/locales/ur/messages.po +++ b/packages/app/src/locales/ur/messages.po @@ -3115,8 +3115,8 @@ msgstr "{displayName} تک رسائی نہیں ہو سکی۔ براہِ کرم msgid "Couldn't reach {displayName} — try again?" msgstr "{displayName} تک رسائی نہیں ہو سکی۔ دوبارہ کوشش کریں؟" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "اشتراکی سرور تک رسائی نہیں ہو سکی — {errorDetail}۔ <0>{cliCommand} status آزمائیں یا <1>.ok/local/last-spawn-error.log دیکھیں۔" +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "اشتراکی سرور تک رسائی نہیں ہو سکی — {errorDetail}۔ <0>{cliCommand} status آزمائیں یا <1>{cliCommand} diagnose bundle کے ذریعے لاگز جمع کریں۔" msgid "Couldn't reach GitHub. Try again?" msgstr "GitHub تک رسائی نہیں ہو سکی۔ دوبارہ کوشش کریں؟" diff --git a/packages/app/src/locales/zh-Hans/messages.po b/packages/app/src/locales/zh-Hans/messages.po index 4bc1606fc..436f9b8d4 100644 --- a/packages/app/src/locales/zh-Hans/messages.po +++ b/packages/app/src/locales/zh-Hans/messages.po @@ -3115,8 +3115,8 @@ msgstr "无法连接 {displayName}。请稍后再试。" msgid "Couldn't reach {displayName} — try again?" msgstr "无法连接 {displayName}。要重试吗？" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "无法连接到协作服务器——{errorDetail}。可尝试 <0>{cliCommand} status，或查看 <1>.ok/local/last-spawn-error.log。" +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "无法连接到协作服务器——{errorDetail}。可尝试 <0>{cliCommand} status，或使用 <1>{cliCommand} diagnose bundle 收集日志。" msgid "Couldn't reach GitHub. Try again?" msgstr "无法连接到 GitHub。要重试吗？" diff --git a/packages/app/src/locales/zh-Hant/messages.po b/packages/app/src/locales/zh-Hant/messages.po index b99c582a5..3726b1985 100644 --- a/packages/app/src/locales/zh-Hant/messages.po +++ b/packages/app/src/locales/zh-Hant/messages.po @@ -3115,8 +3115,8 @@ msgstr "無法連線到 {displayName}。請稍後再試。" msgid "Couldn't reach {displayName} — try again?" msgstr "無法連線到 {displayName}。要再試一次嗎？" -msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or check <1>.ok/local/last-spawn-error.log." -msgstr "無法連上協作伺服器——{errorDetail}。可以試試 <0>{cliCommand} status，或看看 <1>.ok/local/last-spawn-error.log。" +msgid "Couldn't reach collab server — {errorDetail}. Try <0>{cliCommand} status or collect logs with <1>{cliCommand} diagnose bundle." +msgstr "無法連上協作伺服器——{errorDetail}。可以試試 <0>{cliCommand} status，或使用 <1>{cliCommand} diagnose bundle 收集記錄檔。" msgid "Couldn't reach GitHub. Try again?" msgstr "無法連上 GitHub。要再試一次嗎？" diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts index a50b00123..4549c74b3 100644 --- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts +++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts @@ -1,3 +1,4 @@ + import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts index 6fcdc23f2..4fc2c11f9 100644 --- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts +++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts @@ -1,3 +1,4 @@ + import { setTimeout as wait } from 'node:timers/promises'; import { normalizeBridge } from '@inkeep/open-knowledge-core'; import { updateYFragment } from '@tiptap/y-tiptap'; diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.ts index 40b5cdfd6..9f2f6feb6 100644 --- a/packages/app/tests/integration/interior-edit-freshness.test.ts +++ b/packages/app/tests/integration/interior-edit-freshness.test.ts @@ -1,3 +1,4 @@ + import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts index 93666af70..2a085742b 100644 --- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts +++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts @@ -1,3 +1,4 @@ + import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.ts index 867598d34..878d5196b 100644 --- a/packages/app/tests/integration/producer-guard-wiring.test.ts +++ b/packages/app/tests/integration/producer-guard-wiring.test.ts @@ -1,3 +1,4 @@ + import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts index 0970ac12f..4ada3cf26 100644 --- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts +++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts @@ -1,3 +1,4 @@ + import { randomUUID } from 'node:crypto'; import { afterEach, describe, expect, test } from 'vitest'; import { agentWriteMd, createTestServer, type TestServer } from './test-harness'; diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts index b632185e1..cbe2b705f 100644 --- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts +++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts @@ -1,3 +1,4 @@ + import { execFileSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { join } from 'node:path'; diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts index c530ce4ba..89bc9b4d9 100644 --- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts +++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts @@ -1,3 +1,4 @@ + import { randomUUID } from 'node:crypto'; import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest'; import type * as Y from 'yjs'; diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.ts index d3767adc9..a78aeb0e4 100644 --- a/packages/app/tests/integration/undo-after-rollback.test.ts +++ b/packages/app/tests/integration/undo-after-rollback.test.ts @@ -1,3 +1,4 @@ + import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; import * as Y from 'yjs'; diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.ts index b076f955c..e7cc69c3f 100644 --- a/packages/app/tests/integration/undo-recycle-reset.test.ts +++ b/packages/app/tests/integration/undo-recycle-reset.test.ts @@ -1,3 +1,4 @@ + import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.ts index 7935e35f8..9ddcd0730 100644 --- a/packages/app/tests/integration/undo-rederive-survival.test.ts +++ b/packages/app/tests/integration/undo-rederive-survival.test.ts @@ -1,3 +1,4 @@ + import { prependFrontmatter } from '@inkeep/open-knowledge-core'; import { Plugin, PluginKey } from '@tiptap/pm/state'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; diff --git a/packages/app/tests/meta/playwright-cache.test.ts b/packages/app/tests/meta/playwright-cache.test.ts new file mode 100644 index 000000000..12256a0be --- /dev/null +++ b/packages/app/tests/meta/playwright-cache.test.ts @@ -0,0 +1,22 @@ +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { expect, test } from 'vitest'; +import { withTempDir } from '../../../../test-support/temp-dir.test-helper.ts'; + +test('native browser children retain the provisioned cache when HOME is isolated', async () => { + const browser = await import('@playwright/test'); + const executable = browser.chromium.executablePath(); + await withTempDir('ok-browser-child-home-', async (home) => { + const child = spawnSync( + process.execPath, + ['-e', 'process.stdout.write(require("@playwright/test").chromium.executablePath())'], + { + cwd: fileURLToPath(new URL('../..', import.meta.url)), + env: { ...process.env, HOME: home, USERPROFILE: home }, + encoding: 'utf8', + }, + ); + expect(child.status, child.stderr).toBe(0); + expect(child.stdout).toBe(executable); + }); +}); diff --git a/packages/app/vitest.config.ts b/packages/app/vitest.config.ts index 8d6f9dd35..0b1f8a797 100644 --- a/packages/app/vitest.config.ts +++ b/packages/app/vitest.config.ts @@ -1,3 +1,5 @@ +import { homedir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import react from '@vitejs/plugin-react'; import { defineConfig, type ViteUserConfig } from 'vitest/config'; @@ -26,6 +28,13 @@ const serverAuthorityRegistrySetup = fileURLToPath( new URL('../../test-support/server-authority-registry.setup.ts', import.meta.url), ); +const browserCacheRoot = + process.platform === 'darwin' + ? join(homedir(), 'Library', 'Caches') + : process.platform === 'win32' + ? process.env.LOCALAPPDATA || join(homedir(), 'AppData', 'Local') + : process.env.XDG_CACHE_HOME || join(homedir(), '.cache'); + export const appVitestConfig = { ...okVitestBase, publicDir: false, @@ -44,6 +53,11 @@ export const appVitestConfig = { }, test: { ...okVitestBase.test, + env: { + ...okVitestBase.test.env, + PLAYWRIGHT_BROWSERS_PATH: + process.env.PLAYWRIGHT_BROWSERS_PATH || join(browserCacheRoot, 'ms-playwright'), + }, exclude: [ ...okVitestBase.test.exclude, 'public/excalidraw-assets/**', diff --git a/packages/cli/src/commands/bug-report-bundle.test.ts b/packages/cli/src/commands/bug-report-bundle.test.ts index f7adc3bb6..fee430b00 100644 --- a/packages/cli/src/commands/bug-report-bundle.test.ts +++ b/packages/cli/src/commands/bug-report-bundle.test.ts @@ -1,8 +1,18 @@ import { execSync } from 'node:child_process'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; -import { afterEach, describe, expect, test } from 'vitest'; +import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; +import { afterEach, describe, expect, test, vi } from 'vitest'; import { type BundleLogger, collectBugReportLedgerFiles, @@ -24,6 +34,7 @@ function makeTmpDir(prefix = 'ok-bugreport-test-'): string { } afterEach(() => { + vi.unstubAllEnvs(); for (const d of tmpDirs) { if (existsSync(d)) rmSync(d, { recursive: true, force: true }); } @@ -55,6 +66,95 @@ function makeProjectDir(slug = 'bundle-proj'): string { return projectDir; } +function expectedProjectLogsDir(projectDir: string): string { + const key = createHash('sha256').update(realpathSync.native(projectDir)).digest('hex'); + return join(okUserHomeDir(), 'logs', 'projects', key); +} + +const relocatedDiagnostics = [ + [ + 'server-current.jsonl', + '.ok/local/logs/server-current.jsonl', + 'local-logs/server-current.jsonl', + ], + ['server-prev.jsonl', '.ok/local/logs/server-prev.jsonl', 'local-logs/server-prev.jsonl'], + ['last-spawn-error.log', '.ok/local/last-spawn-error.log', 'lockdir/last-spawn-error.log'], +] as const; + +describe('collectStandardBundle — relocated project diagnostics', () => { + test.each(['current', 'legacy', 'both'] as const)( + 'collects %s evidence without duplicate entries, spans, or foreign project data', + async (source) => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectDir = makeProjectDir(); + const currentDir = expectedProjectLogsDir(projectDir); + const foreignDir = expectedProjectLogsDir(makeProjectDir('another-project')); + for (const [filename, legacyPath] of relocatedDiagnostics) { + if (source !== 'legacy') writeAt(currentDir, filename, `current:${filename}\n`); + if (source !== 'current') writeAt(projectDir, legacyPath, `legacy:${filename}\n`); + writeAt(foreignDir, filename, 'foreign-project-decoy\n'); + } + writeAt(foreignDir, 'spans-current.jsonl', 'foreign-span-decoy\n'); + if (source !== 'legacy') writeAt(currentDir, 'spans-current.jsonl', 'excluded-span-decoy\n'); + writeAt(projectDir, '.ok/local/telemetry/spans-current.jsonl', 'excluded-span-decoy\n'); + const outputPath = join(makeTmpDir(), 'report.zip'); + const { zipPath, summary } = await collectStandardBundle({ + projectDir, + redact: false, + outputPath, + }); + const entries = listZipEntries(zipPath); + expect(existsSync(currentDir)).toBe(source !== 'legacy'); + expect(new Set(entries).size).toBe(entries.length); + expect(entries.some((entry) => entry.includes('spans-current'))).toBe(false); + for (const [filename, , entry] of relocatedDiagnostics) { + const origin = source === 'legacy' ? 'legacy' : 'current'; + expect(readZipEntry(zipPath, entry)).toBe(`${origin}:${filename}\n`); + expect(entries.includes(`legacy/${entry}`)).toBe(source === 'both'); + if (source === 'both') + expect(readZipEntry(zipPath, `legacy/${entry}`)).toBe(`legacy:${filename}\n`); + } + for (const entry of summary.files) + expect(readZipEntry(zipPath, entry)).not.toContain('foreign-project-decoy'); + const manifest = JSON.parse(readZipEntry(zipPath, 'MANIFEST.json')); + expect(manifest.files).toEqual(summary.files); + expect(manifest.disciplineVersion).toBe('1.0.0'); + }, + ); + + test('redacts current and retained legacy incidents through the same privacy boundary', async () => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectDir = makeProjectDir(); + const currentDir = expectedProjectLogsDir(projectDir); + const secret = `ghp_${'a'.repeat(40)}`; + const body = `${JSON.stringify({ docName: 'private-note', token: secret })}\n`; + for (const [filename, legacyPath] of relocatedDiagnostics) { + writeAt(currentDir, filename, body); + writeAt(projectDir, legacyPath, body); + } + const outputPath = join(makeTmpDir(), 'report.zip'); + const { zipPath, summary } = await collectStandardBundle({ + projectDir, + redact: true, + outputPath, + }); + for (const [, , entry] of relocatedDiagnostics) { + for (const path of [entry, `legacy/${entry}`]) { + const staged = readZipEntry(zipPath, path); + expect(staged).not.toContain('private-note'); + expect(staged).toContain('doc#'); + expect(staged).not.toContain(secret); + expect(staged).toContain('[REDACTED-GH-PAT]'); + expect(summary.redactions.some((audit) => audit.file === path)).toBe(true); + } + } + }); +}); + describe('collectStandardBundle — diagnostic-report staging count', () => { test('counts what reached the zip, not what the sweep selected', async () => { const projectDir = makeProjectDir(); diff --git a/packages/cli/src/commands/bug-report-bundle.ts b/packages/cli/src/commands/bug-report-bundle.ts index ae723828a..d9403f0be 100644 --- a/packages/cli/src/commands/bug-report-bundle.ts +++ b/packages/cli/src/commands/bug-report-bundle.ts @@ -7,11 +7,14 @@ import { ACP_LAUNCH_FAILURE_LOG, type BundleManifest, type BundleRedaction, + okUserHomeDisplayPath, + PROJECT_DIAGNOSTIC_FILES, REPORT_SENT_MARKER_SUFFIX, REPORT_SIDECAR_BUNDLE_DIR, SERVER_CRASH_LOG, + SPAWN_ERROR_LOG, } from '@inkeep/open-knowledge-core'; -import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; +import { okProjectLogsDir, okUserHomeDir } from '@inkeep/open-knowledge-core/server'; import { withHiddenWindowsConsole } from '@inkeep/open-knowledge-server'; import type { ZipFile } from 'yazl'; import type { DesktopMetadata } from '../diagnose/bundle.ts'; @@ -260,25 +263,28 @@ function collectLockDir(cwd: string): { files: string[] } { const lockDir = join(cwd, '.ok', 'local'); if (!existsSync(lockDir)) return { files: [] }; - const candidates = [ - 'server.lock', - 'last-spawn-error.log', - SERVER_CRASH_LOG, - ACP_LAUNCH_FAILURE_LOG, - ]; + const candidates = ['server.lock', SERVER_CRASH_LOG, ACP_LAUNCH_FAILURE_LOG]; const found = candidates.map((f) => join(lockDir, f)).filter((f) => existsSync(f)); return { files: found }; } -function collectLocalSinkLogs(cwd: string): { files: string[] } { - const logsDir = join(cwd, '.ok', 'local', 'logs'); - if (!existsSync(logsDir)) return { files: [] }; - - const candidates = ['server-current.jsonl', 'server-prev.jsonl']; - const found = candidates.map((f) => join(logsDir, f)).filter((f) => existsSync(f)); - - return { files: found }; +function collectProjectDiagnostics( + projectDir: string, + filenames: readonly string[], + legacyDir: string, +): { files: string[]; legacyFiles: string[] } { + const currentDir = okProjectLogsDir(projectDir); + const files: string[] = []; + const legacyFiles: string[] = []; + for (const filename of filenames) { + const current = join(currentDir, filename); + const legacy = join(legacyDir, filename); + const hasCurrent = existsSync(current); + if (hasCurrent) files.push(current); + if (existsSync(legacy)) (hasCurrent ? legacyFiles : files).push(legacy); + } + return { files, legacyFiles }; } function pseudonymizeDocNames(content: string): { text: string; count: number } { @@ -378,9 +384,20 @@ export async function collectStandardBundle( userLogsDir, ); const { files: lockFiles } = opts.projectDir ? collectLockDir(opts.projectDir) : { files: [] }; - const { files: localSinkFiles } = opts.projectDir - ? collectLocalSinkLogs(opts.projectDir) - : { files: [] }; + const { files: localSinkFiles, legacyFiles: legacyLocalSinkFiles } = opts.projectDir + ? collectProjectDiagnostics( + opts.projectDir, + [PROJECT_DIAGNOSTIC_FILES.logsCurrent, PROJECT_DIAGNOSTIC_FILES.logsPrevious], + join(opts.projectDir, '.ok', 'local', 'logs'), + ) + : { files: [], legacyFiles: [] }; + const { files: spawnErrorFiles, legacyFiles: legacySpawnErrorFiles } = opts.projectDir + ? collectProjectDiagnostics( + opts.projectDir, + [SPAWN_ERROR_LOG], + join(opts.projectDir, '.ok', 'local'), + ) + : { files: [], legacyFiles: [] }; const shipItFiles = opts.shipItLogFiles ?? []; const diagnosticReportFiles = opts.diagnosticReports?.files ?? []; @@ -388,8 +405,8 @@ export async function collectStandardBundle( { logFileCount: logFiles.length, logFilesExcludedByProjectSlug, - lockFileCount: lockFiles.length, - localSinkFileCount: localSinkFiles.length, + lockFileCount: lockFiles.length + spawnErrorFiles.length + legacySpawnErrorFiles.length, + localSinkFileCount: localSinkFiles.length + legacyLocalSinkFiles.length, shipItLogFileCount: shipItFiles.length, diagnosticReportCount: diagnosticReportFiles.length, }, @@ -414,7 +431,7 @@ export async function collectStandardBundle( }); addContentFiles({ zipfile, - files: lockFiles, + files: [...lockFiles, ...spawnErrorFiles], prefix: 'lockdir', redact, revealDocNames, @@ -432,6 +449,26 @@ export async function collectStandardBundle( redactions, logger, }); + addContentFiles({ + zipfile, + files: legacyLocalSinkFiles, + prefix: 'legacy/local-logs', + redact, + revealDocNames, + bundleFiles, + redactions, + logger, + }); + addContentFiles({ + zipfile, + files: legacySpawnErrorFiles, + prefix: 'legacy/lockdir', + redact, + revealDocNames, + bundleFiles, + redactions, + logger, + }); addContentFiles({ zipfile, files: opts.bugReportLedgerFiles ?? [], @@ -560,6 +597,11 @@ export async function collectStandardBundle( '', `Generated: ${manifest.generatedAt}`, `Project: ${projectSlug ?? '(unscoped)'}`, + ...(opts.projectDir + ? [ + `Project diagnostic source: ${okUserHomeDisplayPath('logs', 'projects')} (current product only).`, + ] + : []), `Discipline version: ${manifest.disciplineVersion}`, '', '## Contents', diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts index 8e913af76..0f72f9926 100644 --- a/packages/cli/src/commands/mcp.ts +++ b/packages/cli/src/commands/mcp.ts @@ -31,6 +31,7 @@ export function mcpCommand(getConfig: () => Config): Command { if (opts.port !== undefined) { const timeoutMs = parseSpawnTimeoutEnv(process.env.OK_MCP_SPAWN_TIMEOUT_MS); await startMcpShim({ + projectDir: '', lockDir: '', contentDir: '', portOverride: opts.port, diff --git a/packages/cli/src/diagnose/bundle.test.ts b/packages/cli/src/diagnose/bundle.test.ts index 95c476f06..98907f2e2 100644 --- a/packages/cli/src/diagnose/bundle.test.ts +++ b/packages/cli/src/diagnose/bundle.test.ts @@ -1,4 +1,4 @@ -import { randomUUID } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { existsSync, mkdirSync, @@ -13,7 +13,8 @@ import { } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join, resolve, sep } from 'node:path'; -import { afterEach, describe, expect, onTestFinished, test } from 'vitest'; +import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; +import { afterEach, describe, expect, onTestFinished, test, vi } from 'vitest'; import { ZipFile } from 'yazl'; import type { LanguageMetadata } from '../report-language.ts'; import { @@ -32,6 +33,7 @@ function makeTmpDir(prefix = 'ok-bundle-test-'): string { } afterEach(() => { + vi.unstubAllEnvs(); for (const d of tmpDirs) { if (existsSync(d)) rmSync(d, { recursive: true, force: true }); } @@ -72,6 +74,152 @@ function writeAt(contentDir: string, relPath: string, body: string): void { writeFileSync(full, body); } +function expectedProjectLogsDir(projectDir: string): string { + const key = createHash('sha256').update(realpathSync.native(projectDir)).digest('hex'); + return join(okUserHomeDir(), 'logs', 'projects', key); +} + +const relocatedDiagnostics = [ + ['server-current.jsonl', '.ok/local/logs/server-current.jsonl', 'logs/server-current.jsonl'], + ['server-prev.jsonl', '.ok/local/logs/server-prev.jsonl', 'logs/server-prev.jsonl'], + [ + 'spans-current.jsonl', + '.ok/local/telemetry/spans-current.jsonl', + 'telemetry/spans-current.jsonl', + ], + ['spans-prev.jsonl', '.ok/local/telemetry/spans-prev.jsonl', 'telemetry/spans-prev.jsonl'], + ['last-spawn-error.log', '.ok/local/last-spawn-error.log', 'state/last-spawn-error.log'], +] as const; + +describe('collectBundle — relocated project diagnostics', () => { + test('does not present legacy evidence as current when the current path cannot be copied', async () => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectDir = makeTmpDir(); + mkdirSync(join(expectedProjectLogsDir(projectDir), 'server-current.jsonl'), { + recursive: true, + }); + writeAt(projectDir, '.ok/local/logs/server-current.jsonl', 'older incident\n'); + const collected = await collectBundle({ + contentDir: projectDir, + deps: makeDeterministicDeps(), + }); + onTestFinished(collected.cleanup); + expect(existsSync(join(collected.stagingDir, 'logs/server-current.jsonl'))).toBe(false); + expect( + readFileSync(join(collected.stagingDir, 'legacy/logs/server-current.jsonl'), 'utf8'), + ).toBe('older incident\n'); + }); + + test.each([ + ['stable', '~/.ok/logs/projects'], + ['beta', '~/.ok-beta/logs/projects'], + ] as const)( + 'records the %s diagnostic root without exposing the account home', + async (channel, directory) => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + vi.stubEnv('OK_CHANNEL', channel); + const contentDir = makeTmpDir(); + const collected = await collectBundle({ contentDir, deps: makeDeterministicDeps() }); + onTestFinished(collected.cleanup); + expect(collected.manifest.telemetry.localSink.directory).toBe(directory); + expect(JSON.stringify(collected.manifest)).not.toContain(home); + }, + ); + + test.each(['current', 'legacy', 'both'] as const)( + 'collects %s evidence without overwriting incidents or reading another project', + async (source) => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectDir = makeTmpDir(); + const contentDir = join(projectDir, 'knowledge'); + mkdirSync(contentDir); + const currentDir = expectedProjectLogsDir(projectDir); + const foreignDir = expectedProjectLogsDir(makeTmpDir()); + for (const [filename, legacyPath] of relocatedDiagnostics) { + if (source !== 'legacy') writeAt(currentDir, filename, `current:${filename}\n`); + if (source !== 'current') writeAt(projectDir, legacyPath, `legacy:${filename}\n`); + writeAt(contentDir, legacyPath, 'content-subdir-decoy\n'); + writeAt(foreignDir, filename, 'foreign-project-decoy\n'); + } + const collected = await collectBundle({ + contentDir, + projectDir, + deps: makeDeterministicDeps(), + }); + onTestFinished(collected.cleanup); + for (const [filename, , entry] of relocatedDiagnostics) { + const origin = source === 'legacy' ? 'legacy' : 'current'; + expect(readFileSync(join(collected.stagingDir, entry), 'utf8')).toBe( + `${origin}:${filename}\n`, + ); + const legacyEntry = join(collected.stagingDir, 'legacy', entry); + expect(existsSync(legacyEntry)).toBe(source === 'both'); + if (source === 'both') + expect(readFileSync(legacyEntry, 'utf8')).toBe(`legacy:${filename}\n`); + } + expect(existsSync(currentDir)).toBe(source !== 'legacy'); + expect(collected.manifest.schemaVersion).toBe(2); + for (const file of collected.manifest.files) { + const body = readFileSync(join(collected.stagingDir, file.path), 'utf8'); + expect(body).not.toContain('foreign-project-decoy'); + expect(body).not.toContain('content-subdir-decoy'); + } + const outputPath = join(makeTmpDir(), 'bundle.zip'); + await writeBundle({ collected, outputPath }); + const entries = await readZipEntries(outputPath); + expect(new Set(entries).size).toBe(entries.length); + expect(entries.sort()).toEqual( + ['manifest.json', ...collected.manifest.files.map((file) => file.path)].sort(), + ); + }, + ); + + test('redacts current and legacy diagnostics and counts both log/span histories', async () => { + const home = makeTmpDir(); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectDir = makeTmpDir(); + const contentDir = join(projectDir, 'knowledge'); + mkdirSync(contentDir); + const currentDir = expectedProjectLogsDir(projectDir); + const secret = `ghp_${'a'.repeat(40)}`; + const body = `${JSON.stringify({ docName: 'private-note', path: `${contentDir}/note.md`, token: secret })}\n`; + const diagnostics = relocatedDiagnostics.filter( + ([filename]) => filename !== 'server-prev.jsonl' && filename !== 'spans-prev.jsonl', + ); + for (const [filename, legacyPath] of diagnostics) { + writeAt(currentDir, filename, body); + writeAt(projectDir, legacyPath, body); + } + const collected = await collectBundle({ + contentDir, + projectDir, + redact: true, + scrubSecrets: true, + deps: makeDeterministicDeps(), + }); + onTestFinished(collected.cleanup); + for (const [, , entry] of diagnostics) { + for (const path of [entry, `legacy/${entry}`]) { + const staged = readFileSync(join(collected.stagingDir, path), 'utf8'); + expect(staged).not.toContain(contentDir); + expect(staged).toContain('/note.md'); + expect(staged).not.toContain(secret); + expect( + collected.manifest.redaction.secretScrub?.redactions.some((audit) => audit.file === path), + ).toBe(true); + } + } + expect(collected.summary.docNameCount).toBe(4); + }); +}); + describe('collectBundle — diagnostic-report staging count', () => { test('counts what was staged, not what the sweep selected', async () => { const contentDir = makeTmpDir(); diff --git a/packages/cli/src/diagnose/bundle.ts b/packages/cli/src/diagnose/bundle.ts index 4995aed6c..982d49de8 100644 --- a/packages/cli/src/diagnose/bundle.ts +++ b/packages/cli/src/diagnose/bundle.ts @@ -17,16 +17,26 @@ import { basename, dirname, join, relative, resolve, sep } from 'node:path'; import type { BundleRedaction as SecretScrubEntry } from '@inkeep/open-knowledge-core'; import { ACP_LAUNCH_FAILURE_LOG, + okUserHomeDisplayPath, + PROJECT_DIAGNOSTIC_FILES, REPORT_SIDECAR_BUNDLE_DIR, SERVER_CRASH_LOG, SERVER_EXIT_LOG, + SPAWN_ERROR_LOG, } from '@inkeep/open-knowledge-core'; import { DEFAULT_LOGS_MAX_BYTES, DEFAULT_SPANS_MAX_BYTES, + projectSpawnErrorLogPath, resolveConfigPath, } from '@inkeep/open-knowledge-core/server'; -import { withHiddenWindowsConsole } from '@inkeep/open-knowledge-server'; +import { + logsCurrentPath, + logsPreviousPath, + spansCurrentPath, + spansPreviousPath, + withHiddenWindowsConsole, +} from '@inkeep/open-knowledge-server'; import { parse as parseYaml } from 'yaml'; import { ZipFile } from 'yazl'; import { @@ -97,6 +107,7 @@ interface BundleManifest { enabled: boolean; spansMaxBytes: number; logsMaxBytes: number; + directory: string; }; otlpPushEnabled: boolean; }; @@ -159,29 +170,13 @@ export interface WriteBundleOpts { const TELEMETRY_REL = ['.ok', 'local', 'telemetry'] as const; const LOGS_REL = ['.ok', 'local', 'logs'] as const; const LOSS_CAPTURE_REL = ['.ok', 'local', 'loss-capture'] as const; -const SPANS_CURRENT = 'spans-current.jsonl'; -const SPANS_PREVIOUS = 'spans-prev.jsonl'; -const LOGS_CURRENT = 'server-current.jsonl'; -const LOGS_PREVIOUS = 'server-prev.jsonl'; +const SPANS_CURRENT = PROJECT_DIAGNOSTIC_FILES.spansCurrent; +const SPANS_PREVIOUS = PROJECT_DIAGNOSTIC_FILES.spansPrevious; +const LOGS_CURRENT = PROJECT_DIAGNOSTIC_FILES.logsCurrent; +const LOGS_PREVIOUS = PROJECT_DIAGNOSTIC_FILES.logsPrevious; const LOSS_CURRENT = 'loss-current.jsonl'; const LOSS_PREVIOUS = 'loss-prev.jsonl'; -function spansCurrentPath(projectDir: string): string { - return join(projectDir, ...TELEMETRY_REL, SPANS_CURRENT); -} - -function spansPreviousPath(projectDir: string): string { - return join(projectDir, ...TELEMETRY_REL, SPANS_PREVIOUS); -} - -function logsCurrentPath(projectDir: string): string { - return join(projectDir, ...LOGS_REL, LOGS_CURRENT); -} - -function logsPreviousPath(projectDir: string): string { - return join(projectDir, ...LOGS_REL, LOGS_PREVIOUS); -} - function lossCaptureCurrentPath(projectDir: string): string { return join(projectDir, ...LOSS_CAPTURE_REL, LOSS_CURRENT); } @@ -191,10 +186,6 @@ function lossCapturePreviousPath(projectDir: string): string { } export const _pathHelpersForTests = { - spansCurrentPath, - spansPreviousPath, - logsCurrentPath, - logsPreviousPath, lossCaptureCurrentPath, lossCapturePreviousPath, }; @@ -416,6 +407,20 @@ function stageFileIfPresent(srcPath: string, destPath: string): boolean { } } +function stageDiagnosticFile( + currentPath: string, + legacyPath: string, + stagingDir: string, + entryPath: string, +): void { + const hasCurrent = existsSync(currentPath); + stageFileIfPresent(currentPath, join(stagingDir, entryPath)); + stageFileIfPresent( + legacyPath, + hasCurrent ? join(stagingDir, 'legacy', entryPath) : join(stagingDir, entryPath), + ); +} + function walkStagedFiles(stagingDir: string): string[] { const results: string[] = []; const recurse = (dir: string): void => { @@ -501,13 +506,30 @@ export async function collectBundle(opts: CollectBundleOpts): Promise' : contentDir, }, telemetry: { - localSink, + localSink: { ...localSink, directory: okUserHomeDisplayPath('logs', 'projects') }, otlpPushEnabled: isOtlpPushEnabled(), }, redaction, diff --git a/packages/cli/src/diagnose/path-parity.test.ts b/packages/cli/src/diagnose/path-parity.test.ts index 1017e847d..ef3790b41 100644 --- a/packages/cli/src/diagnose/path-parity.test.ts +++ b/packages/cli/src/diagnose/path-parity.test.ts @@ -1,9 +1,3 @@ -import { - logsCurrentPath as serverLogsCurrentPath, - logsPreviousPath as serverLogsPreviousPath, - spansCurrentPath as serverSpansCurrentPath, - spansPreviousPath as serverSpansPreviousPath, -} from '@inkeep/open-knowledge-server'; import { describe, expect, test } from 'vitest'; import { lossCaptureCurrentPath as serverLossCaptureCurrentPath, @@ -15,10 +9,6 @@ describe('CLI bundle path helpers — parity with the server writers', () => { const fixtures = ['/tmp/content', '/Users/dev/projects/foo', '/var/data/with spaces/dir']; const pairs = { - spansCurrentPath: serverSpansCurrentPath, - spansPreviousPath: serverSpansPreviousPath, - logsCurrentPath: serverLogsCurrentPath, - logsPreviousPath: serverLogsPreviousPath, lossCaptureCurrentPath: serverLossCaptureCurrentPath, lossCapturePreviousPath: serverLossCapturePreviousPath, } satisfies Record string>; diff --git a/packages/cli/src/diagnose/staging-coverage.test.ts b/packages/cli/src/diagnose/staging-coverage.test.ts index d2d025a1c..766292f58 100644 --- a/packages/cli/src/diagnose/staging-coverage.test.ts +++ b/packages/cli/src/diagnose/staging-coverage.test.ts @@ -1,12 +1,14 @@ -import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { afterEach, describe, expect, test } from 'vitest'; +import { okProjectLogsDir } from '@inkeep/open-knowledge-core/server'; +import { afterEach, describe, expect, test, vi } from 'vitest'; import { collectBundle, SECRET_SCRUB_EXTENSIONS } from './bundle.ts'; import { CONTENT_SUBDIRS_MASKED } from './bundle-redact.ts'; const tmpDirs: string[] = []; afterEach(() => { + vi.unstubAllEnvs(); for (const d of tmpDirs) rmSync(d, { recursive: true, force: true }); tmpDirs.length = 0; }); @@ -18,6 +20,7 @@ describe('staged-content coverage', () => { process: 'masked', 'diagnostic-reports': 'masked', state: 'masked-separately', + legacy: 'masked-separately', extra: 'raw-bytes-by-design', } as const; @@ -68,6 +71,22 @@ describe('staged-content coverage', () => { const contentDir = mkdtempSync(resolve(tmpdir(), 'ok-staging-coverage-')); tmpDirs.push(contentDir); writeFileSync(join(contentDir, 'note.md'), '# note\n'); + const home = mkdtempSync(resolve(tmpdir(), 'ok-staging-coverage-home-')); + tmpDirs.push(home); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const currentDir = okProjectLogsDir(contentDir); + mkdirSync(currentDir, { recursive: true }); + const diagnosticFamilies = [ + ['server-current.jsonl', join(contentDir, '.ok', 'local', 'logs')], + ['spans-current.jsonl', join(contentDir, '.ok', 'local', 'telemetry')], + ['last-spawn-error.log', join(contentDir, '.ok', 'local')], + ] as const; + for (const [filename, legacyDir] of diagnosticFamilies) { + mkdirSync(legacyDir, { recursive: true }); + writeFileSync(join(currentDir, filename), '{"source":"current"}\n'); + writeFileSync(join(legacyDir, filename), '{"source":"legacy"}\n'); + } const reportsDir = mkdtempSync(resolve(tmpdir(), 'ok-staging-coverage-reports-')); tmpDirs.push(reportsDir); @@ -119,6 +138,7 @@ describe('staged-content coverage', () => { expect(produced).toContain('diagnostic-reports'); expect(produced).toContain('process'); expect(produced).toContain('extra'); + expect(produced).toContain('legacy'); for (const dir of produced) { expect(Object.keys(TOP_LEVEL), `staged directory "${dir}" is undeclared`).toContain(dir); } diff --git a/packages/cli/src/mcp/server.ts b/packages/cli/src/mcp/server.ts index c26b1e2cb..eba60d438 100644 --- a/packages/cli/src/mcp/server.ts +++ b/packages/cli/src/mcp/server.ts @@ -190,7 +190,7 @@ export async function startGlobalMcpServer( displayName: id.displayName, clientName: id.clientInfo?.name ?? id.displayName, colorSeed: id.colorSeed, - resolveWsUrl: async () => resolveMcpKeepaliveWsUrl({ lockDir, contentDir: projectDir }, ''), + resolveWsUrl: async () => resolveMcpKeepaliveWsUrl({ lockDir }, ''), log: (msg) => stderr.write(`[mcp] keepalive[${projectDir}]: ${msg}\n`), }); keepalivesByProject.set(projectDir, handle); @@ -241,6 +241,7 @@ export async function startGlobalMcpServer( const projectDir = r.projectDir; const config = await resolveConfigForCwd(projectDir); const mcpUrl = await resolveMcpHttpUrl({ + projectDir, lockDir: getLocalDir(projectDir), contentDir: resolveContentDir(config, projectDir), envAutoStart, diff --git a/packages/cli/src/mcp/shim.test.ts b/packages/cli/src/mcp/shim.test.ts index 9191cae27..a908f4564 100644 --- a/packages/cli/src/mcp/shim.test.ts +++ b/packages/cli/src/mcp/shim.test.ts @@ -1,15 +1,24 @@ -import { mkdirSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, writeSync } from 'node:fs'; import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { setTimeout as wait } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; import { formatSpawnAttemptHeader, LOCAL_DIR, OK_DIR, SPAWN_ERROR_LOG, } from '@inkeep/open-knowledge-core'; -import { AutoStartDisabledError, type ServerLockMetadata } from '@inkeep/open-knowledge-server'; +import { okProjectLogsDir } from '@inkeep/open-knowledge-core/server'; +import { + AutoStartDisabledError, + prepareSpawnProjectLogsDir, + type ServerLockMetadata, +} from '@inkeep/open-knowledge-server'; import type { JSONRPCMessage } from '@modelcontextprotocol/sdk/types.js'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { @@ -83,15 +92,145 @@ const liveLock: ServerLockMetadata = { describe('MCP stdio shim server resolution', () => { let tmp: string; + let fixture: string; let lockDir: string; beforeEach(async () => { - tmp = await mkdtemp(resolve(tmpdir(), 'ok-mcp-shim-')); + fixture = await mkdtemp(resolve(tmpdir(), 'ok-mcp-shim-')); + tmp = resolve(fixture, 'project'); + mkdirSync(tmp); + vi.stubEnv('HOME', resolve(fixture, 'home')); + vi.stubEnv('USERPROFILE', resolve(fixture, 'home')); lockDir = resolve(tmp, OK_DIR, LOCAL_DIR); }); afterEach(async () => { - await rm(tmp, { recursive: true, force: true }); + await rm(fixture, { recursive: true, force: true }); + }); + + test.each(['foreign-lease', 'blocked-namespace'])( + 'default spawn preparation keeps MCP stdout clean (%s)', + async (scenario) => { + const registryPath = resolve(fixture, 'authority.sqlite'); + const namespace = await prepareSpawnProjectLogsDir(tmp, { + authorityRegistryPath: registryPath, + }); + if (scenario === 'foreign-lease') { + const database = new DatabaseSync(registryPath); + try { + database + .prepare('INSERT INTO server_authority VALUES (?, ?, ?, ?, ?, ?, ?)') + .run( + randomUUID(), + randomUUID(), + process.pid, + 'future-protocol:v2', + tmp, + JSON.stringify({ kind: 'tree', path: tmp, excluded: [] }), + new Date().toISOString(), + ); + } finally { + database.close(); + } + } else { + rmSync(namespace, { recursive: true }); + writeFileSync(namespace, 'namespace obstruction'); + } + const code = ` + const { resolveMcpHttpUrl } = await import(${JSON.stringify(new URL('./shim.ts', import.meta.url).href)}); + process.argv[1] = ${JSON.stringify(fileURLToPath(new URL('../../dist/cli.mjs', import.meta.url)))}; + try { + await resolveMcpHttpUrl({ + projectDir: ${JSON.stringify(tmp)}, contentDir: ${JSON.stringify(tmp)}, + lockDir: ${JSON.stringify(lockDir)}, authorityRegistryPath: ${JSON.stringify(registryPath)}, + readLock: () => null, isAlive: () => false, + spawn: () => { throw new Error('fixture-spawn-failure'); }, + }); + throw new Error('Unexpected successful spawn'); + } catch (error) { + process.stderr.write('RESULT=' + JSON.stringify({ message: error.message }) + String.fromCharCode(10)); + } + `; + const child = spawnSync( + process.execPath, + ['--import', 'tsx', '--conditions=development', '--input-type=module', '-e', code], + { + cwd: fileURLToPath(new URL('../..', import.meta.url)), + env: { + ...process.env, + LOG_LEVEL: 'info', + OK_CONSOLE_LEVEL: 'info', + NODE_ENV: 'production', + }, + encoding: 'utf8', + timeout: 15_000, + windowsHide: true, + }, + ); + expect(child.status, child.stderr).toBe(0); + const line = child.stderr.split('\n').find((value) => value.startsWith('RESULT=')); + expect(line).toBeDefined(); + const result = JSON.parse(line?.slice('RESULT='.length) ?? '{}') as { message: string }; + expect({ stdout: child.stdout, message: result.message }).toEqual({ + stdout: '', + message: expect.stringContaining( + scenario === 'blocked-namespace' + ? 'spawn failed: could not prepare server-start diagnostics:' + : 'spawn failed: fixture-spawn-failure', + ), + }); + expect(result.message).toContain('Run `ok start` in the target project'); + }, + ); + + test('async diagnostic preparation failures retain their cause and server-start guidance', async () => { + const cause = new Error('fixture registry unavailable'); + const spawn = vi.fn(); + const error: unknown = await resolveMcpHttpUrl({ + projectDir: tmp, + lockDir, + contentDir: tmp, + readLock: () => null, + isAlive: () => false, + openErrorLog: async () => { + throw cause; + }, + spawn, + }).catch((value: unknown) => value); + expect(error).toBeInstanceOf(Error); + expect((error as Error).cause).toBe(cause); + expect((error as Error).message).toContain('spawn failed:'); + expect((error as Error).message).toContain('Run `ok start` in the target project'); + expect(spawn).not.toHaveBeenCalled(); + }); + + test('captures startup stderr outside the project even when content lives in a subdirectory', async () => { + const contentDir = resolve(tmp, 'knowledge'); + mkdirSync(contentDir); + const stderrPath = resolve(okProjectLogsDir(tmp), SPAWN_ERROR_LOG); + + await expect( + resolveMcpHttpUrl({ + projectDir: tmp, + authorityRegistryPath: resolve(fixture, 'authority.sqlite'), + lockDir, + contentDir, + readLock: () => null, + isAlive: () => false, + spawn: (( + _cmd: string, + _args: readonly string[], + options: { stdio: readonly [string, string, number] }, + ) => { + writeSync(options.stdio[2], 'captured startup cause\n'); + throw new Error('startup failed'); + }) as never, + }), + ).rejects.toThrow('spawn failed: startup failed stderr:\ncaptured startup cause'); + + expect(readFileSync(stderrPath, 'utf8')).toContain('captured startup cause'); + expect(existsSync(resolve(lockDir, SPAWN_ERROR_LOG))).toBe(false); + expect(existsSync(okProjectLogsDir(contentDir))).toBe(false); }); test('draining lock waits for the predecessor to exit, then spawns', async () => { @@ -100,6 +239,7 @@ describe('MCP stdio shim server resolution', () => { const calls: string[] = []; const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => { reads += 1; @@ -129,6 +269,7 @@ describe('MCP stdio shim server resolution', () => { let reads = 0; const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => { reads += 1; @@ -149,6 +290,7 @@ describe('MCP stdio shim server resolution', () => { test('live lock resolves directly to the /mcp HTTP URL', async () => { const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => liveLock, isAlive: (pid) => pid === liveLock.pid, @@ -164,6 +306,7 @@ describe('MCP stdio shim server resolution', () => { const lockWithUrl: ServerLockMetadata = { ...liveLock, url: 'http://localhost:4999' }; const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => lockWithUrl, isAlive: (pid) => pid === lockWithUrl.pid, @@ -179,6 +322,7 @@ describe('MCP stdio shim server resolution', () => { const lockWithBadUrl: ServerLockMetadata = { ...liveLock, url: 'not a url' }; const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => lockWithBadUrl, isAlive: (pid) => pid === lockWithBadUrl.pid, @@ -201,6 +345,7 @@ describe('MCP stdio shim server resolution', () => { const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => { pollCount += 1; @@ -233,6 +378,7 @@ describe('MCP stdio shim server resolution', () => { test('auto-start opt-out turns missing server into a short diagnostic', async () => { const err: unknown = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, envAutoStart: '0', readLock: () => null, @@ -249,6 +395,7 @@ describe('MCP stdio shim server resolution', () => { vi.stubEnv('OK_CHANNEL', 'beta'); const err: unknown = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, envAutoStart: '0', readLock: () => null, @@ -264,6 +411,8 @@ describe('MCP stdio shim server resolution', () => { async (kind) => { const err: unknown = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, + authorityRegistryPath: resolve(fixture, 'authority.sqlite'), contentDir: resolve(tmp, kind === 'missing' ? 'missing' : 'x'.repeat(32768)), timeoutMs: 1000, pollIntervalMs: 1, @@ -282,6 +431,7 @@ describe('MCP stdio shim server resolution', () => { test('valid port override bypasses discovery and targets the default loopback host', async () => { const url = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, portOverride: '6789', readLock: () => { @@ -300,6 +450,7 @@ describe('MCP stdio shim server resolution', () => { await expect( resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, portOverride: 'not-a-port', spawn: (() => { @@ -313,6 +464,7 @@ describe('MCP stdio shim server resolution', () => { await expect( resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => null, isAlive: () => false, @@ -330,15 +482,17 @@ describe('MCP stdio shim server resolution', () => { }); test('the default reader quotes only the current attempt', async () => { - mkdirSync(lockDir, { recursive: true }); + const stderrPath = resolve(okProjectLogsDir(tmp), SPAWN_ERROR_LOG); + mkdirSync(okProjectLogsDir(tmp), { recursive: true }); writeFileSync( - resolve(lockDir, SPAWN_ERROR_LOG), + stderrPath, `${formatSpawnAttemptHeader(new Date('2026-08-25T10:00:00.000Z'), 11)}EADDRINUSE from before\n` + `${formatSpawnAttemptHeader(new Date('2026-08-25T11:00:00.000Z'), 12)}the real cause\n`, ); const err = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => null, isAlive: () => false, @@ -361,15 +515,17 @@ describe('MCP stdio shim server resolution', () => { }); test('the default reader bounds one huge attempt to a tail', async () => { - mkdirSync(lockDir, { recursive: true }); + const stderrPath = resolve(okProjectLogsDir(tmp), SPAWN_ERROR_LOG); + mkdirSync(okProjectLogsDir(tmp), { recursive: true }); const oldest = 'FIRST-LINE-MARKER\n'; writeFileSync( - resolve(lockDir, SPAWN_ERROR_LOG), + stderrPath, `${formatSpawnAttemptHeader(new Date('2026-08-25T11:00:00.000Z'), 12)}${oldest}${'x'.repeat(20_000)}`, ); const err = await resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => null, isAlive: () => false, @@ -397,6 +553,7 @@ describe('MCP stdio shim server resolution', () => { await expect( resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => null, isAlive: () => false, @@ -422,6 +579,7 @@ describe('MCP stdio shim server resolution', () => { await expect( resolveMcpHttpUrl({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => null, isAlive: () => false, @@ -450,7 +608,6 @@ describe('MCP stdio shim server resolution', () => { resolveMcpKeepaliveWsUrl( { lockDir, - contentDir: tmp, readLock: () => liveLock, isAlive: () => true, }, @@ -462,7 +619,6 @@ describe('MCP stdio shim server resolution', () => { resolveMcpKeepaliveWsUrl( { lockDir, - contentDir: tmp, readLock: () => ({ ...liveLock, url: 'http://localhost:4999' }), isAlive: () => true, }, @@ -474,7 +630,6 @@ describe('MCP stdio shim server resolution', () => { resolveMcpKeepaliveWsUrl( { lockDir, - contentDir: tmp, readLock: () => liveLock, isAlive: () => false, }, @@ -486,7 +641,6 @@ describe('MCP stdio shim server resolution', () => { resolveMcpKeepaliveWsUrl( { lockDir, - contentDir: tmp, portOverride: '5123', readLock: () => null, isAlive: () => false, @@ -591,6 +745,7 @@ describe('startMcpShim lifecycle', () => { await expect( startMcpShim({ lockDir, + projectDir: tmp, contentDir: tmp, readLock: () => liveLock, isAlive: () => true, diff --git a/packages/cli/src/mcp/shim.ts b/packages/cli/src/mcp/shim.ts index e103b505c..8c3b21872 100644 --- a/packages/cli/src/mcp/shim.ts +++ b/packages/cli/src/mcp/shim.ts @@ -1,23 +1,22 @@ import { type ChildProcess, spawn as nativeSpawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { closeSync, existsSync, mkdirSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import type { Readable, Writable } from 'node:stream'; import { setTimeout as wait } from 'node:timers/promises'; import { clientVersionHeaders, currentCliCommandName, DEFAULT_SERVER_HOST, - SPAWN_ERROR_LOG, sliceLastSpawnAttempt, } from '@inkeep/open-knowledge-core'; import { startKeepalive as defaultStartKeepalive } from '@inkeep/open-knowledge-core/keepalive'; -import { openSpawnErrorLog } from '@inkeep/open-knowledge-core/server'; +import { openSpawnErrorLog, projectSpawnErrorLogPath } from '@inkeep/open-knowledge-core/server'; import { AutoStartDisabledError, isProcessAlive as defaultIsProcessAlive, lockBaseUrl, MCP_CONNECTION_ID_HEADER, + prepareSpawnProjectLogsDir, RUNTIME_VERSION, readServerLock, type ServerLockMetadata, @@ -72,6 +71,8 @@ export function parseSpawnTimeoutEnv(raw: string | undefined): number | undefine } interface ResolveMcpHttpUrlOptions { + projectDir: string; + authorityRegistryPath?: string; lockDir: string; contentDir: string; portOverride?: string; @@ -81,7 +82,7 @@ interface ResolveMcpHttpUrlOptions { isAlive?: (pid: number) => boolean; sleep?: (ms: number) => Promise; readErrorLog?: (path: string) => string; - openErrorLog?: (path: string) => number; + openErrorLog?: (path: string) => number | Promise; closeFd?: (fd: number) => void; timeoutMs?: number; pollIntervalMs?: number; @@ -195,7 +196,13 @@ export async function resolveMcpHttpUrl(opts: ResolveMcpHttpUrlOptions): Promise const spawnFn = opts.spawn ?? nativeSpawn; const readErrorLog = opts.readErrorLog ?? readErrorLogDefault; const openErrorLog = - opts.openErrorLog ?? ((path: string) => openSpawnErrorLog(path, process.pid)); + opts.openErrorLog ?? + (async (path: string) => { + await prepareSpawnProjectLogsDir(opts.projectDir, { + authorityRegistryPath: opts.authorityRegistryPath, + }); + return openSpawnErrorLog(path, process.pid); + }); const closeFd = opts.closeFd ?? closeSync; const timeoutMs = opts.timeoutMs ?? DEFAULT_SPAWN_TIMEOUT_MS; const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; @@ -241,8 +248,16 @@ export async function resolveMcpHttpUrl(opts: ResolveMcpHttpUrlOptions): Promise } if (!existsSync(opts.lockDir)) mkdirSync(opts.lockDir, { recursive: true }); - const stderrPath = join(opts.lockDir, SPAWN_ERROR_LOG); - const stderrFd = openErrorLog(stderrPath); + const stderrPath = projectSpawnErrorLogPath(opts.projectDir); + let stderrFd: number; + try { + stderrFd = await openErrorLog(stderrPath); + } catch (err) { + throw new Error( + `spawn failed: could not prepare server-start diagnostics: ${err instanceof Error ? err.message : String(err)}\n${serverStartRecovery()}`, + { cause: err }, + ); + } let child: ChildProcess | undefined; let asyncSpawnError: string | undefined; const self = resolveSelfSpawn(); @@ -295,7 +310,7 @@ export async function resolveMcpHttpUrl(opts: ResolveMcpHttpUrlOptions): Promise } export function resolveMcpKeepaliveWsUrl( - opts: ResolveMcpHttpUrlOptions, + opts: Pick, endpointUrl: string, ): string | undefined { if (opts.portOverride !== undefined) return wsUrlFromMcpEndpoint(endpointUrl); diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.ts index f4cc8a286..2a597ec3d 100644 --- a/packages/core/src/bridge/compose.ts +++ b/packages/core/src/bridge/compose.ts @@ -1,3 +1,4 @@ + import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts'; export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved'; diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts index e7e63f43d..ffd8fce06 100644 --- a/packages/core/src/bridge/doc-boundary-space.ts +++ b/packages/core/src/bridge/doc-boundary-space.ts @@ -1,3 +1,4 @@ + import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts'; import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts'; diff --git a/packages/core/src/config/schema.ts b/packages/core/src/config/schema.ts index 5a4849c67..c8d2c44e9 100644 --- a/packages/core/src/config/schema.ts +++ b/packages/core/src/config/schema.ts @@ -570,7 +570,7 @@ export const ConfigSchema = z.looseObject({ reload: 'live', defaultScope: 'project', description: - 'Write local diagnostic spans + logs under .ok/local/ for `ok diagnose bundle`. Local-only — never leaves the machine until you run bundle. Set false for sensitive workspaces. Shared across collaborators.', + 'Write per-project diagnostic spans + logs under the user-level OpenKnowledge logs directory for `ok diagnose bundle`. Files stay local unless shared, including in a support bundle. Set false for sensitive workspaces. Shared across collaborators.', }) .default(true), spans: z diff --git a/packages/core/src/constants/lifecycle.ts b/packages/core/src/constants/lifecycle.ts index 1060ff7a8..eed719d1f 100644 --- a/packages/core/src/constants/lifecycle.ts +++ b/packages/core/src/constants/lifecycle.ts @@ -4,6 +4,14 @@ export const DEFAULT_SIGTERM_POLL_MS = 200; export const SPAWN_ERROR_LOG = 'last-spawn-error.log'; +export const PROJECT_DIAGNOSTIC_FILES = { + logsCurrent: 'server-current.jsonl', + logsPrevious: 'server-prev.jsonl', + spansCurrent: 'spans-current.jsonl', + spansPrevious: 'spans-prev.jsonl', + spawnError: SPAWN_ERROR_LOG, +} as const; + export const SPAWN_ERROR_LOG_MAX_BYTES = 256 * 1024; export const SPAWN_ATTEMPT_MARKER = '=== spawn attempt '; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8b6766ae4..c619da2be 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -483,6 +483,7 @@ export { DEFAULT_SIGTERM_GRACE_MS, DEFAULT_SIGTERM_POLL_MS, formatSpawnAttemptHeader, + PROJECT_DIAGNOSTIC_FILES, SERVER_CRASH_LOG, SERVER_EXIT_LOG, SPAWN_ATTEMPT_MARKER, diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts index fc267cbea..fb0ba5a7c 100644 --- a/packages/core/src/markdown/comment-promoter.ts +++ b/packages/core/src/markdown/comment-promoter.ts @@ -1,3 +1,4 @@ + import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts index 861003fb4..ae8cb390b 100644 --- a/packages/core/src/markdown/merged-walker.ts +++ b/packages/core/src/markdown/merged-walker.ts @@ -1,3 +1,4 @@ + import type { Nodes, Parent, Root } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts index 4d3250d3a..8be632ba4 100644 --- a/packages/core/src/markdown/safe-url.ts +++ b/packages/core/src/markdown/safe-url.ts @@ -1,3 +1,4 @@ + export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const; const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|'); diff --git a/packages/core/src/server.ts b/packages/core/src/server.ts index 619a08a4c..901d374bd 100644 --- a/packages/core/src/server.ts +++ b/packages/core/src/server.ts @@ -53,4 +53,10 @@ export { withFileLock, withFileLockSync, } from './util/file-lock.ts'; +export { + isProjectLogsKey, + okProjectLogsDir, + projectLogsKeyForCanonicalPath, + projectSpawnErrorLogPath, +} from './util/project-logs-dir.ts'; export { openSpawnErrorLog } from './util/spawn-error-log-open.ts'; diff --git a/packages/core/src/util/project-logs-dir.test.ts b/packages/core/src/util/project-logs-dir.test.ts new file mode 100644 index 000000000..5d4a92cb0 --- /dev/null +++ b/packages/core/src/util/project-logs-dir.test.ts @@ -0,0 +1,61 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { afterEach, expect, test } from 'vitest'; +import { okUserHomeDir } from '../constants/ok-user-home.ts'; +import { okProjectLogsDir } from './project-logs-dir.ts'; + +const directories: string[] = []; + +function fixture(): { project: string; home: string; root: string } { + const root = mkdtempSync(join(tmpdir(), 'ok-project-logs-')); + directories.push(root); + const project = join(root, 'project'); + mkdirSync(project); + return { project, home: join(root, 'home'), root }; +} + +afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('uses an opaque project namespace below the product user logs without creating directories', () => { + const { project, home } = fixture(); + const path = okProjectLogsDir(project, home); + + expect(dirname(path)).toBe(join(okUserHomeDir(home), 'logs', 'projects')); + expect(basename(path)).toMatch(/^[a-f0-9]{64}$/); + expect(okProjectLogsDir(join(project, 'child', '..'), home)).toBe(path); + expect(existsSync(home)).toBe(false); +}); + +test('isolates two projects with the same basename', () => { + const first = fixture(); + const second = fixture(); + + expect(okProjectLogsDir(first.project, first.home)).not.toBe( + okProjectLogsDir(second.project, first.home), + ); +}); + +test('an existing symlink spelling uses the same project namespace', () => { + const { project, home, root } = fixture(); + const alias = join(root, 'alias'); + symlinkSync(project, alias, process.platform === 'win32' ? 'junction' : 'dir'); + + expect(okProjectLogsDir(alias, home)).toBe(okProjectLogsDir(project, home)); +}); + +test('a missing project still has a read-only namespaced location', () => { + const { project, home, root } = fixture(); + const missing = join(root, 'missing'); + const path = okProjectLogsDir(missing, home); + + expect(dirname(path)).toBe(join(okUserHomeDir(home), 'logs', 'projects')); + expect(basename(path)).toMatch(/^[a-f0-9]{64}$/); + expect(path).not.toBe(okProjectLogsDir(project, home)); + expect(existsSync(missing)).toBe(false); + expect(existsSync(home)).toBe(false); +}); diff --git a/packages/core/src/util/project-logs-dir.ts b/packages/core/src/util/project-logs-dir.ts new file mode 100644 index 000000000..c2b281390 --- /dev/null +++ b/packages/core/src/util/project-logs-dir.ts @@ -0,0 +1,26 @@ +import { createHash } from 'node:crypto'; +import { realpathSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { SPAWN_ERROR_LOG } from '../constants/lifecycle.ts'; +import { okUserHomeDir } from '../constants/ok-user-home.ts'; + +export function projectLogsKeyForCanonicalPath(projectPath: string): string { + return createHash('sha256').update(projectPath).digest('hex'); +} + +export function isProjectLogsKey(value: string): boolean { + return /^[a-f0-9]{64}$/.test(value); +} + +export function projectSpawnErrorLogPath(projectDir: string, home?: string): string { + return join(okProjectLogsDir(projectDir, home), SPAWN_ERROR_LOG); +} + +export function okProjectLogsDir(projectDir: string, home?: string): string { + let projectPath = resolve(projectDir); + try { + projectPath = realpathSync.native(projectPath); + } catch {} + const key = projectLogsKeyForCanonicalPath(projectPath); + return join(okUserHomeDir(home), 'logs', 'projects', key); +} diff --git a/packages/core/src/util/spawn-error-log-open.test.ts b/packages/core/src/util/spawn-error-log-open.test.ts index 5cef4502e..b741c03d1 100644 --- a/packages/core/src/util/spawn-error-log-open.test.ts +++ b/packages/core/src/util/spawn-error-log-open.test.ts @@ -9,7 +9,7 @@ import { writeSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { afterEach, describe, expect, test } from 'vitest'; import { SPAWN_ATTEMPT_MARKER, SPAWN_ERROR_LOG_MAX_BYTES } from '../constants/lifecycle.ts'; import { openSpawnErrorLog } from './spawn-error-log-open.ts'; @@ -39,6 +39,21 @@ function attempt(path: string, childOutput: string, pid = 4242): void { } describe('openSpawnErrorLog', () => { + test('creates a missing log directory before capturing the attempt', () => { + const path = join( + dirname(scratchPath()), + 'logs', + 'projects', + 'project', + 'last-spawn-error.log', + ); + attempt(path, 'nested failure\n'); + + expect(readFileSync(path, 'utf8')).toContain('nested failure'); + if (process.platform !== 'win32') expect(statSync(dirname(path)).mode & 0o777).toBe(0o700); + expect(statSync(path).mode & 0o777).toBe(0o600); + }); + test('creates the log when absent, with the header and the output', () => { const path = scratchPath(); attempt(path, 'boom\n'); diff --git a/packages/core/src/util/spawn-error-log-open.ts b/packages/core/src/util/spawn-error-log-open.ts index a7e8a6ca2..adf37e48b 100644 --- a/packages/core/src/util/spawn-error-log-open.ts +++ b/packages/core/src/util/spawn-error-log-open.ts @@ -1,7 +1,9 @@ -import { fchmodSync, openSync, statSync, writeSync } from 'node:fs'; +import { fchmodSync, mkdirSync, openSync, statSync, writeSync } from 'node:fs'; +import { dirname } from 'node:path'; import { formatSpawnAttemptHeader, spawnErrorLogOpenMode } from '../constants/lifecycle.ts'; export function openSpawnErrorLog(path: string, spawningPid: number): number { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); let existingSize: number | undefined; try { existingSize = statSync(path).size; diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts index ece0322b1..3d9048f93 100644 --- a/packages/desktop/src/main/index.ts +++ b/packages/desktop/src/main/index.ts @@ -61,7 +61,6 @@ import { projectWorktreeCreateResult, resolveDesktopProductName, ServerInfoSuccessSchema, - SPAWN_ERROR_LOG, TERMINAL_CLIS, type TerminalCli, type UninstallFeedbackAnswers, @@ -73,7 +72,7 @@ import type { OkTerminalDockStateWriteResult, OkTerminalRestartSnapshot, } from '@inkeep/open-knowledge-core/desktop-bridge'; -import { openSpawnErrorLog } from '@inkeep/open-knowledge-core/server'; +import { openSpawnErrorLog, projectSpawnErrorLogPath } from '@inkeep/open-knowledge-core/server'; import { parseSkillDir } from '@inkeep/open-knowledge-core/skills-catalog'; import { assertGitAvailable, @@ -91,6 +90,7 @@ import { normalizeFsPath, ONBOARDING_BUNDLE_IDS, prepareSingleFileOpen, + prepareSpawnProjectLogsDir, type ResolvedSkillHost, RUNTIME_VERSION, readBundleDecision, @@ -1410,14 +1410,15 @@ function ensureWindowManager() { ); } } - const spawnErrorLogPath = join(lockDir, SPAWN_ERROR_LOG); + const spawnErrorLogPath = projectSpawnErrorLogPath(projectRoot); let spawnErrorLogFd: number; try { + await prepareSpawnProjectLogsDir(projectRoot); spawnErrorLogFd = openSpawnErrorLog(spawnErrorLogPath, process.pid); } catch (err) { throw Object.assign( new Error( - `spawnCliServer: failed to open spawn-error log fd at ${spawnErrorLogPath}: ${ + `spawnCliServer: server start failed while preparing diagnostics at ${spawnErrorLogPath}: ${ err instanceof Error ? err.message : String(err) }`, ), diff --git a/packages/desktop/src/main/window-manager.ts b/packages/desktop/src/main/window-manager.ts index 43f69747b..9b57d888b 100644 --- a/packages/desktop/src/main/window-manager.ts +++ b/packages/desktop/src/main/window-manager.ts @@ -7,10 +7,10 @@ import { DEFAULT_SIGTERM_POLL_MS, type DesktopProductName, desktopChannelLabel, - SPAWN_ERROR_LOG, sliceLastSpawnAttempt, } from '@inkeep/open-knowledge-core'; import type { KeepaliveHandle } from '@inkeep/open-knowledge-core/keepalive'; +import { projectSpawnErrorLogPath } from '@inkeep/open-knowledge-core/server'; import { assessLockHolder, getLocalDir, @@ -1581,6 +1581,7 @@ export class WindowManager { } if (this.deps.spawnDetachedServer) { + const spawnErrorLogPath = projectSpawnErrorLogPath(projectPath); const reactShellDistDir = dirname(this.deps.rendererEntryPath); const handle = await this.deps.spawnDetachedServer({ contentDir: projectPath, @@ -1606,6 +1607,7 @@ export class WindowManager { pid: handle.pid, exit: childExit, lockDir, + spawnErrorLogPath, deadlineMs: waitedDeadlineMs, spawnLabel: 'spawn', childExited, @@ -2070,6 +2072,7 @@ export class WindowManager { const tempProjectDir = createEphemeralProjectDir(opts.contentDir); const lockDir = getLocalDir(tempProjectDir); + const spawnErrorLogPath = projectSpawnErrorLogPath(tempProjectDir); const reactShellDistDir = dirname(this.deps.rendererEntryPath); let handle: Awaited>; @@ -2115,6 +2118,7 @@ export class WindowManager { pid: handle.pid, exit: childExit, lockDir, + spawnErrorLogPath, deadlineMs: waitedDeadlineMs, spawnLabel: 'ephemeral spawn', childExited, @@ -2324,24 +2328,29 @@ export class WindowManager { pid: number; exit: { code: number | null; signal: string | null } | null; lockDir: string; + spawnErrorLogPath: string; deadlineMs: number; spawnLabel: string; childExited: boolean | undefined; }): Error { - const { pid, exit, lockDir, deadlineMs, spawnLabel, childExited } = opts; + const { pid, exit, lockDir, spawnErrorLogPath, deadlineMs, spawnLabel, childExited } = opts; const STDERR_TAIL_BYTES = 8192; let stderrTail: string | undefined; try { - const attempt = sliceLastSpawnAttempt( - readFileSync(join(lockDir, SPAWN_ERROR_LOG), 'utf-8'), - ).trim(); + const attempt = sliceLastSpawnAttempt(readFileSync(spawnErrorLogPath, 'utf-8')).trim(); stderrTail = attempt.length > STDERR_TAIL_BYTES ? `…${attempt.slice(-STDERR_TAIL_BYTES)}` : attempt; } catch (readErr) { const code = (readErr as NodeJS.ErrnoException).code; if (code !== 'ENOENT' && code !== 'ENOTDIR') { this.deps.log?.warn( - { event: 'desktop-spawn-error-log-read-failed', err: readErr, code, lockDir }, + { + event: 'desktop-spawn-error-log-read-failed', + err: readErr, + code, + lockDir, + spawnErrorLogPath, + }, '[window-manager] could not read the spawn error log', ); } diff --git a/packages/desktop/tests/main/window-manager-ephemeral.test.ts b/packages/desktop/tests/main/window-manager-ephemeral.test.ts index 0a6995f8a..888cd2b12 100644 --- a/packages/desktop/tests/main/window-manager-ephemeral.test.ts +++ b/packages/desktop/tests/main/window-manager-ephemeral.test.ts @@ -1,6 +1,21 @@ +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; +import { formatSpawnAttemptHeader, SPAWN_ERROR_LOG } from '@inkeep/open-knowledge-core'; +import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; import { getLocalDir } from '@inkeep/open-knowledge-server'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { beforeEach, describe, expect, onTestFinished, test, vi } from 'vitest'; import type { ShowGateRegistry } from '../../src/main/show-gate.ts'; import type { WindowChromeKind } from '../../src/main/window-chrome.ts'; import { @@ -321,6 +336,58 @@ describe('createEphemeralWindow', () => { expect(env.windows).toHaveLength(0); }); + test('ephemeral cleanup preserves the user-namespace stderr tail', async () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), 'ok-ephemeral-stderr-')); + onTestFinished(() => { + vi.unstubAllEnvs(); + rmSync(fixtureRoot, { recursive: true, force: true }); + }); + const home = join(fixtureRoot, 'home'); + const contentDir = join(fixtureRoot, 'notes'); + const projectTarget = join(fixtureRoot, 'project-target'); + const projectDir = join(fixtureRoot, 'project-link'); + mkdirSync(contentDir); + mkdirSync(projectTarget); + symlinkSync(projectTarget, projectDir, process.platform === 'win32' ? 'junction' : 'dir'); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectHash = createHash('sha256').update(realpathSync.native(projectDir)).digest('hex'); + const logsDir = join(okUserHomeDir(home), 'logs', 'projects', projectHash); + mkdirSync(logsDir, { recursive: true }); + const stderr = 'Error: ephemeral server failed before binding'; + const spawnErrorLogPath = join(logsDir, SPAWN_ERROR_LOG); + writeFileSync(spawnErrorLogPath, `${formatSpawnAttemptHeader(new Date(), 42001)}${stderr}\n`); + env.deps.createEphemeralProjectDir = () => projectDir; + env.deps.spawnSingleFileServer = async () => + heldChild(env.deps, 42001, () => ({ code: 1, signal: null })); + env.deps.readServerLock = () => null; + env.deps.spawnLockPollDeadlineMs = 0; + env.deps.removeDir = async (dir) => { + env.removedDirs.push(dir); + await rm(dir, { recursive: true, force: true }); + await rm(projectTarget, { recursive: true, force: true }); + }; + + const wm = new WindowManager(env.deps); + await expect( + wm.createEphemeralWindow({ + canonicalFilePath: join(contentDir, 'todo.md'), + contentDir, + docName: 'todo', + }), + ).rejects.toMatchObject({ + kind: 'spawn-lock-timeout', + exitCode: 1, + stderrTail: stderr, + message: expect.stringContaining('--- stderr ---'), + }); + expect(env.removedDirs).toEqual([projectDir]); + expect(existsSync(projectDir)).toBe(false); + expect(existsSync(projectTarget)).toBe(false); + expect(existsSync(spawnErrorLogPath)).toBe(true); + expect(env.windows).toHaveLength(0); + }); + test('a live child that binds after the startup deadline is not SIGTERMd', async () => { env.publishLock = false; env.deps.setTimeout = (cb: () => void) => { diff --git a/packages/desktop/tests/main/window-manager.test.ts b/packages/desktop/tests/main/window-manager.test.ts index 014ef45e5..02b734e2f 100644 --- a/packages/desktop/tests/main/window-manager.test.ts +++ b/packages/desktop/tests/main/window-manager.test.ts @@ -1,9 +1,15 @@ +import { createHash } from 'node:crypto'; import { EventEmitter } from 'node:events'; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; -import { DEFAULT_SERVER_HOST, formatSpawnAttemptHeader } from '@inkeep/open-knowledge-core'; +import { + DEFAULT_SERVER_HOST, + formatSpawnAttemptHeader, + SPAWN_ERROR_LOG, +} from '@inkeep/open-knowledge-core'; +import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; import { getMachineId } from '@inkeep/open-knowledge-server'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { failedOpenHolder, promptHolderStop } from '../../src/main/holder-stop-prompt.ts'; @@ -3068,6 +3074,7 @@ describe('WindowManager', () => { const spawnStderrRoots: string[] = []; afterEach(() => { + vi.unstubAllEnvs(); while (spawnStderrRoots.length > 0) { const root = spawnStderrRoots.pop(); if (root) rmSync(root, { recursive: true, force: true }); @@ -3075,10 +3082,17 @@ describe('WindowManager', () => { }); function projectWithSpawnErrorLog(contents: string): string { - const root = mkdtempSync(join(tmpdir(), 'ok-spawn-stderr-')); - spawnStderrRoots.push(root); - mkdirSync(join(root, '.ok', 'local'), { recursive: true }); - writeFileSync(join(root, '.ok', 'local', 'last-spawn-error.log'), contents); + const fixtureRoot = mkdtempSync(join(tmpdir(), 'ok-spawn-stderr-')); + spawnStderrRoots.push(fixtureRoot); + const root = join(fixtureRoot, 'project'); + const home = join(fixtureRoot, 'home'); + mkdirSync(root); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + const projectHash = createHash('sha256').update(realpathSync.native(root)).digest('hex'); + const logsDir = join(okUserHomeDir(home), 'logs', 'projects', projectHash); + mkdirSync(logsDir, { recursive: true }); + writeFileSync(join(logsDir, SPAWN_ERROR_LOG), contents); return root; } diff --git a/packages/server/src/boot-project-log-retention.test.ts b/packages/server/src/boot-project-log-retention.test.ts new file mode 100644 index 000000000..ed8f0c5e2 --- /dev/null +++ b/packages/server/src/boot-project-log-retention.test.ts @@ -0,0 +1,201 @@ +import { execFileSync } from 'node:child_process'; +import { + closeSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + rmSync, + statSync, + utimesSync, + writeFileSync, + writeSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { SPAWN_ERROR_LOG } from '@inkeep/open-knowledge-core'; +import { okProjectLogsDir, openSpawnErrorLog } from '@inkeep/open-knowledge-core/server'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts'; +import { type BootedServer, bootServer } from './boot.ts'; +import { ConfigSchema } from './config/schema.ts'; +import { loggerFactory } from './logger.ts'; +import { prepareProjectLogsDir } from './project-logs-retention.ts'; +import * as serverAuthority from './server-authority.ts'; +import { shutdownTelemetry } from './telemetry.ts'; + +let root: string; +let projectDir: string; +let registryPath: string; +let namespace: string; +let stderrFd: number | undefined; +let booted: BootedServer | undefined; +let idleShutdown: (() => Promise) | undefined; + +beforeEach(() => { + root = realpathSync.native(mkdtempSync(join(tmpdir(), 'ok-boot-log-retention-'))); + const home = join(root, 'home'); + mkdirSync(home); + vi.stubEnv('HOME', home); + vi.stubEnv('USERPROFILE', home); + vi.stubEnv('CODEX_HOME', join(home, '.codex')); + vi.stubEnv('CLAUDE_CONFIG_DIR', join(home, '.claude')); + vi.stubEnv('OK_CHANNEL', 'stable'); + vi.stubEnv('OK_DISABLE_LOCAL_SINK', '0'); + vi.stubEnv('OTEL_SDK_DISABLED', 'true'); + vi.stubEnv('LOG_LEVEL', 'silent'); + vi.stubEnv('OK_CONSOLE_LEVEL', 'silent'); + vi.stubEnv('OK_FILE_LEVEL', 'debug'); + vi.stubEnv('GIT_CONFIG_NOSYSTEM', '1'); + vi.stubEnv('GIT_CONFIG_GLOBAL', join(home, 'gitconfig')); + vi.stubEnv('GIT_TERMINAL_PROMPT', '0'); + projectDir = join(root, 'project'); + registryPath = join(root, 'injected-authority', 'registry.sqlite'); + namespace = okProjectLogsDir(projectDir); + loggerFactory.reset(); + idleShutdown = undefined; +}); + +afterEach(async () => { + try { + await booted?.destroy(); + } finally { + booted = undefined; + if (stderrFd !== undefined) closeSync(stderrFd); + stderrFd = undefined; + try { + await shutdownTelemetry(); + await loggerFactory.flushAllFileSinks(); + } finally { + loggerFactory.reset(); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + rmSync(root, { recursive: true, force: true }); + } + } +}); + +async function startBoot(localSinksEnabled: boolean): Promise { + execFileSync('git', ['init', '--initial-branch=main', projectDir], { stdio: 'pipe' }); + configureTestGitRepository(projectDir); + mkdirSync(join(projectDir, '.ok')); + writeFileSync( + join(projectDir, '.ok', 'config.yml'), + `telemetry:\n localSink:\n enabled: ${localSinksEnabled}\n`, + ); + writeFileSync(join(projectDir, '.ok', '.gitignore'), ''); + const originalGuard = serverAuthority.withServerAuthorityProjectDirs; + vi.spyOn(serverAuthority, 'withServerAuthorityProjectDirs').mockImplementation((fn, path) => { + if (path !== registryPath) throw new Error('Boot retention did not use its injected registry'); + return originalGuard(fn, path); + }); + booted = await bootServer({ + host: '127.0.0.1', + config: ConfigSchema.parse({}), + contentDir: projectDir, + projectDir, + authorityRegistryPath: registryPath, + configHomedirOverride: join(root, 'home'), + port: 0, + quiet: true, + gitEnabled: false, + idleShutdownHandler: (destroy) => { + idleShutdown = destroy; + return destroy; + }, + }); + await booted.ready; + await loggerFactory.flushAllFileSinks(); + return booted; +} + +test.each([ + [true, false], + [false, false], + [true, true], + [false, true], +])( + 'real boot preserves final stderr through authority retirement (local sinks=%s, idle=%s)', + async (localSinksEnabled, idle) => { + const stderrPath = join(namespace, SPAWN_ERROR_LOG); + stderrFd = openSpawnErrorLog(stderrPath, process.pid); + const observations: Array<{ + beforeRelease: number; + afterPrune: boolean; + stderr: string | null; + }> = []; + const originalAcquire = serverAuthority.acquireServerAuthority; + vi.spyOn(serverAuthority, 'acquireServerAuthority').mockImplementation((options) => { + if (options.registryPath !== registryPath) + throw new Error('Boot owner did not use its injected registry'); + const handle = originalAcquire(options); + return { + scope: handle.scope, + release: () => { + const beforeRelease = statSync(namespace).mtimeMs; + handle.release(); + const currentProject = join(root, 'pruner-project'); + mkdirSync(currentProject, { recursive: true }); + prepareProjectLogsDir(currentProject, { authorityRegistryPath: registryPath }); + const afterPrune = existsSync(namespace); + if (stderrFd === undefined) throw new Error('Owned stderr descriptor absent'); + writeSync(stderrFd, 'final stderr after authority retirement\n'); + observations.push({ + beforeRelease, + afterPrune, + stderr: existsSync(stderrPath) ? readFileSync(stderrPath, 'utf8') : null, + }); + }, + }; + }); + expect(existsSync(registryPath)).toBe(false); + const server = await startBoot(localSinksEnabled); + const registry = new DatabaseSync(registryPath, { readOnly: true }); + try { + expect(registry.prepare('SELECT count(*) AS count FROM server_authority').get()?.count).toBe( + 1, + ); + } finally { + registry.close(); + } + const old = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + if (localSinksEnabled) { + for (const name of [ + 'server-current.jsonl', + 'server-prev.jsonl', + 'spans-current.jsonl', + 'spans-prev.jsonl', + ]) { + if (!existsSync(join(namespace, name))) writeFileSync(join(namespace, name), ''); + } + } else { + expect(readdirSync(namespace)).toEqual([SPAWN_ERROR_LOG]); + } + for (const name of readdirSync(namespace)) utimesSync(join(namespace, name), old, old); + utimesSync(namespace, old, old); + const inactivityCutoff = Date.now() - 7 * 24 * 60 * 60 * 1000; + if (idle) { + if (!idleShutdown) throw new Error('Idle shutdown callback absent'); + await idleShutdown(); + } else { + await server.destroy(); + } + booted = undefined; + expect(observations).toHaveLength(1); + expect(observations[0]?.beforeRelease).toBeGreaterThanOrEqual(inactivityCutoff); + expect(observations[0]?.afterPrune).toBe(true); + expect(observations[0]?.stderr).toContain('final stderr after authority retirement'); + }, +); + +test('disabled local sinks without existing stderr do not create a namespace during real boot or destroy', async () => { + expect(existsSync(namespace)).toBe(false); + const server = await startBoot(false); + expect(existsSync(namespace)).toBe(false); + await server.destroy(); + booted = undefined; + expect(existsSync(namespace)).toBe(false); +}); diff --git a/packages/server/src/boot-skill-read-stall.test.ts b/packages/server/src/boot-skill-read-stall.test.ts index 74bf174f0..19265ef7c 100644 --- a/packages/server/src/boot-skill-read-stall.test.ts +++ b/packages/server/src/boot-skill-read-stall.test.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { performance } from 'node:perf_hooks'; import { setTimeout as sleep } from 'node:timers/promises'; import { expect, test, vi } from 'vitest'; -import { bootServer } from './boot.ts'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { ConfigSchema } from './config/schema.ts'; import { scanGlobalInPlaceSkillsAsync } from './in-place-skills.ts'; diff --git a/packages/server/src/boot.test.ts b/packages/server/src/boot.test.ts index f9109350c..0b8d6ca87 100644 --- a/packages/server/src/boot.test.ts +++ b/packages/server/src/boot.test.ts @@ -14,8 +14,8 @@ import { SimpleSpanProcessor, } from '@opentelemetry/sdk-trace-base'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts'; -import { bootServer } from './boot.ts'; import { getBootTimings } from './boot-timings.ts'; import { ConfigSchema } from './config/schema.ts'; import { getLogger } from './logger.ts'; diff --git a/packages/server/src/boot.ts b/packages/server/src/boot.ts index 3585dba3b..b207bbe4a 100644 --- a/packages/server/src/boot.ts +++ b/packages/server/src/boot.ts @@ -1,4 +1,4 @@ -import { existsSync, readdirSync, statSync } from 'node:fs'; +import { existsSync, lstatSync, readdirSync, statSync } from 'node:fs'; import type { Server as HttpServer } from 'node:http'; import { homedir } from 'node:os'; import { resolve } from 'node:path'; @@ -18,7 +18,7 @@ import { type ServerRuntimeConfig, SKILLS_STORE_DIRNAME, } from '@inkeep/open-knowledge-core'; -import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; +import { okProjectLogsDir, okUserHomeDir } from '@inkeep/open-knowledge-core/server'; import { resolveGitDirDetailed } from '@inkeep/open-knowledge-core/shadow-repo-layout'; import { context, propagation } from '@opentelemetry/api'; import sirv from 'sirv'; @@ -33,7 +33,7 @@ import { createAssetServeMiddleware } from './asset-serve-middleware.ts'; import { bootElapsedMs, recordBootPhase, startBootTimings } from './boot-timings.ts'; import type { Config } from './config/schema.ts'; import { installCrashCapture } from './crash-capture.ts'; -import { normalizeFsPath } from './fs-traced.ts'; +import { normalizeFsPath, tracedUtimesSync } from './fs-traced.ts'; import { assertGitAvailable, type GitDetected, @@ -59,6 +59,7 @@ import { getLogger, loggerFactory, type PinoLogger } from './logger.ts'; import { createMcpHttpHandler } from './mcp-http.ts'; import { mountMcpAndApi, type ReadinessState } from './mcp-mount.ts'; import { MissingOkConfigError } from './missing-ok-config-error.ts'; +import { prepareProjectLogsDir } from './project-logs-retention.ts'; import { createProjectRuntime, type ProjectRuntime } from './project-runtime.ts'; import { createServer, @@ -204,10 +205,12 @@ export async function bootServer(opts: BootServerOptions): Promise startBootTimings(); const sinkProjectDir = opts.projectDir ?? opts.contentDir; + const projectLogsDir = okProjectLogsDir(sinkProjectDir); const localSinkConfig = resolveLocalSinkConfig({ projectDir: sinkProjectDir, }); if (localSinkConfig) { + prepareProjectLogsDir(sinkProjectDir, { authorityRegistryPath: opts.authorityRegistryPath }); const denylist = localSinkConfig.telemetry.attributeDenylist; const redactPaths: string[] = []; for (const key of denylist) { @@ -237,7 +240,9 @@ export async function bootServer(opts: BootServerOptions): Promise const startupTraceparent = process.env.OK_STARTUP_TRACEPARENT; const bootSpan = () => - withSpan('ok.boot', { attributes: spanAttributes }, async () => bootServerInner(opts)); + withSpan('ok.boot', { attributes: spanAttributes }, async () => + bootServerInner(opts, projectLogsDir), + ); const runBoot = (): Promise => { if (startupTraceparent) { try { @@ -269,7 +274,10 @@ export async function bootServer(opts: BootServerOptions): Promise return booted; } -async function bootServerInner(opts: BootServerOptions): Promise { +async function bootServerInner( + opts: BootServerOptions, + projectLogsDir: string, +): Promise { const skipAutoInit = opts.skipAutoInit ?? false; const log = opts.log ?? getLogger('boot'); @@ -712,6 +720,13 @@ async function bootServerInner(opts: BootServerOptions): Promise { destroy = async (reason: ServerExitReason = 'unspecified'): Promise => { if (destroyed) return; destroyed = true; + try { + const directory = lstatSync(projectLogsDir); + if (directory.isDirectory() && !directory.isSymbolicLink()) { + const now = new Date(); + tracedUtimesSync(projectLogsDir, now, now); + } + } catch {} log.info({ reason, pid: process.pid, lockDir }, `[server] shutdown initiated (${reason})`); readinessState = 'draining'; const errors: unknown[] = []; diff --git a/packages/server/src/collab-message-limit.test.ts b/packages/server/src/collab-message-limit.test.ts index 642cd31f2..e534812d1 100644 --- a/packages/server/src/collab-message-limit.test.ts +++ b/packages/server/src/collab-message-limit.test.ts @@ -8,7 +8,7 @@ import * as encoding from 'lib0/encoding'; import { afterAll, describe, expect, test } from 'vitest'; import { WebSocket as WsClient } from 'ws'; import { messageYjsUpdate } from 'y-protocols/sync'; -import { type BootedServer, bootServer } from './boot.ts'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { getMetrics, resetMetrics } from './metrics.ts'; async function bootTestServer(): Promise<{ booted: BootedServer; contentDir: string }> { diff --git a/packages/server/src/console-discipline.test.ts b/packages/server/src/console-discipline.test.ts index e3c036f73..e33666d5a 100644 --- a/packages/server/src/console-discipline.test.ts +++ b/packages/server/src/console-discipline.test.ts @@ -118,7 +118,7 @@ describe('console discipline (server sources)', () => { expect( violations, `Raw console.* call found in packages/server/src — console output never reaches the pino file sink ` + - `(.ok/local/logs/server-current.jsonl) that bug-report bundles collect, and the packaged desktop app ` + + `(user-level project logs) that bug-report bundles collect, and the packaged desktop app ` + `runs the server detached with no persistent stdio. Use getLogger(name) from './logger.ts' ` + `(warn/error still reach the terminal through the console stream). The only sanctioned console shapes ` + `are the structured telemetry-event channel (console.(JSON.stringify({ event: ... }))) and the ` + diff --git a/packages/server/src/crash-capture.test.ts b/packages/server/src/crash-capture.test.ts index 44bdb73ce..dc37cf0d7 100644 --- a/packages/server/src/crash-capture.test.ts +++ b/packages/server/src/crash-capture.test.ts @@ -1,26 +1,33 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { mkdir, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { SERVER_CRASH_LOG } from '@inkeep/open-knowledge-core'; -import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { buildCrashRecord, type CrashCaptureHandle, installCrashCapture, writeCrashArtifacts, } from './crash-capture.ts'; +import { logsCurrentPath } from './telemetry-file-sink.ts'; +let fixtureDir: string; let projectDir: string; const handles: CrashCaptureHandle[] = []; beforeEach(async () => { - projectDir = await mkdtemp(join(tmpdir(), 'ok-crash-capture-')); + fixtureDir = await mkdtemp(join(tmpdir(), 'ok-crash-capture-')); + projectDir = join(fixtureDir, 'project'); + await mkdir(projectDir); + vi.stubEnv('HOME', join(fixtureDir, 'home')); + vi.stubEnv('USERPROFILE', join(fixtureDir, 'home')); }); afterEach(async () => { for (const handle of handles.splice(0)) handle.uninstall(); - await rm(projectDir, { recursive: true, force: true }); + vi.unstubAllEnvs(); + await rm(fixtureDir, { recursive: true, force: true }); }); function emitMonitor(err: unknown, origin: string): void { @@ -28,7 +35,7 @@ function emitMonitor(err: unknown, origin: string): void { } const crashJsonPath = () => join(projectDir, '.ok', 'local', SERVER_CRASH_LOG); -const logsPath = () => join(projectDir, '.ok', 'local', 'logs', 'server-current.jsonl'); +const logsPath = () => logsCurrentPath(projectDir); describe('buildCrashRecord', () => { test('captures name, message, stack, pid, uptime from an Error', () => { @@ -61,6 +68,8 @@ describe('writeCrashArtifacts', () => { expect(crashJson.pid).toBe(process.pid); const lines = readFileSync(logsPath(), 'utf-8').trim().split('\n'); + if (process.platform !== 'win32') + expect(statSync(dirname(logsPath())).mode & 0o777).toBe(0o700); expect(lines).toHaveLength(1); const logLine = JSON.parse(lines[0]); expect(logLine.level).toBe(60); @@ -70,7 +79,7 @@ describe('writeCrashArtifacts', () => { }); test('appends to an existing log sink file instead of truncating it', () => { - mkdirSync(join(projectDir, '.ok', 'local', 'logs'), { recursive: true }); + mkdirSync(dirname(logsPath()), { recursive: true }); writeFileSync(logsPath(), '{"level":30,"msg":"earlier"}\n'); writeCrashArtifacts(projectDir, buildCrashRecord(new Error('later'), 'uncaughtException')); const lines = readFileSync(logsPath(), 'utf-8').trim().split('\n'); @@ -99,6 +108,8 @@ describe('writeCrashArtifacts', () => { test('swallows write failures instead of throwing', () => { mkdirSync(join(projectDir, '.ok'), { recursive: true }); writeFileSync(join(projectDir, '.ok', 'local'), 'not a directory'); + mkdirSync(dirname(dirname(logsPath())), { recursive: true }); + writeFileSync(dirname(logsPath()), 'not a directory'); expect(() => writeCrashArtifacts(projectDir, buildCrashRecord(new Error('x'), 'uncaughtException')), ).not.toThrow(); diff --git a/packages/server/src/crash-capture.ts b/packages/server/src/crash-capture.ts index af4fc3ea2..0c9cce994 100644 --- a/packages/server/src/crash-capture.ts +++ b/packages/server/src/crash-capture.ts @@ -34,7 +34,7 @@ export function writeCrashArtifacts(projectDir: string, record: CrashRecord): vo } catch {} try { const logsPath = logsCurrentPath(projectDir); - mkdirSync(dirname(logsPath), { recursive: true }); + mkdirSync(dirname(logsPath), { recursive: true, mode: 0o700 }); const line = { level: 60, time: Date.parse(record.timestamp), diff --git a/packages/server/src/fs-traced.ts b/packages/server/src/fs-traced.ts index 1bc88ca1e..d18ddf49d 100644 --- a/packages/server/src/fs-traced.ts +++ b/packages/server/src/fs-traced.ts @@ -15,6 +15,7 @@ import { rmSync, symlinkSync, unlinkSync, + utimesSync, writeFileSync, writeSync, } from 'node:fs'; @@ -63,6 +64,11 @@ export const tracedChmodSync: typeof chmodSync = (path, mode) => chmodSync(path, mode), ); +export const tracedUtimesSync: typeof utimesSync = (path, atime, mtime) => + withSpanSync('fs.utimesSync', { attributes: buildAttrs('utimesSync', String(path)) }, () => + utimesSync(path, atime, mtime), + ); + function byteLength(data: string | Uint8Array | ArrayBufferView): number { if (typeof data === 'string') return Buffer.byteLength(data, 'utf-8'); if (data instanceof Uint8Array) return data.byteLength; diff --git a/packages/server/src/git-preflight-boot.test.ts b/packages/server/src/git-preflight-boot.test.ts index 05bfa0b3c..a06b171ac 100644 --- a/packages/server/src/git-preflight-boot.test.ts +++ b/packages/server/src/git-preflight-boot.test.ts @@ -10,7 +10,7 @@ import { SimpleSpanProcessor, } from '@opentelemetry/sdk-trace-base'; import { afterEach, beforeEach, describe, expect, test } from 'vitest'; -import { bootServer } from './boot.ts'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { ConfigSchema } from './config/schema.ts'; import { type GitDetected, diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 4d60b58a7..719dd6d3c 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -497,6 +497,7 @@ export { ensureProjectGit, ProjectGitInitError, } from './project-git.ts'; +export { prepareProjectLogsDir, prepareSpawnProjectLogsDir } from './project-logs-retention.ts'; export { CONFLICT_MARKER_RE, containsConflictMarkers, diff --git a/packages/server/src/keepalive-presence-bootstrap.test.ts b/packages/server/src/keepalive-presence-bootstrap.test.ts index 3597db582..3136a1d75 100644 --- a/packages/server/src/keepalive-presence-bootstrap.test.ts +++ b/packages/server/src/keepalive-presence-bootstrap.test.ts @@ -5,8 +5,8 @@ import { setTimeout as wait } from 'node:timers/promises'; import { OK_DIR } from '@inkeep/open-knowledge-core'; import { afterAll, describe, expect, test } from 'vitest'; import { WebSocket as WsClient } from 'ws'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { toBroadcasterKey } from './agent-id.ts'; -import { type BootedServer, bootServer } from './boot.ts'; import { parseKeepaliveIdentity } from './mcp-mount.ts'; async function poll( diff --git a/packages/server/src/keepalive-presence-cleanup.test.ts b/packages/server/src/keepalive-presence-cleanup.test.ts index 9adcf3663..5aa29fbfb 100644 --- a/packages/server/src/keepalive-presence-cleanup.test.ts +++ b/packages/server/src/keepalive-presence-cleanup.test.ts @@ -5,8 +5,8 @@ import { setTimeout as wait } from 'node:timers/promises'; import { OK_DIR } from '@inkeep/open-knowledge-core'; import { afterAll, describe, expect, test } from 'vitest'; import { WebSocket as WsClient } from 'ws'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { toBroadcasterKey } from './agent-id.ts'; -import { type BootedServer, bootServer } from './boot.ts'; async function poll( read: () => T, diff --git a/packages/server/src/lint/write-okf-schemas-wiring.test.ts b/packages/server/src/lint/write-okf-schemas-wiring.test.ts index 8a520e7df..2b79e2bba 100644 --- a/packages/server/src/lint/write-okf-schemas-wiring.test.ts +++ b/packages/server/src/lint/write-okf-schemas-wiring.test.ts @@ -8,7 +8,7 @@ import { okfAdvertisedSchemaMappings, } from '@inkeep/open-knowledge-core'; import { afterEach, beforeEach, describe, expect, test } from 'vitest'; -import { type BootedServer, bootServer } from '../boot.ts'; +import { type BootedServer, bootServer } from '../../../../test-support/boot-server.test-helper.ts'; import { resetOkfSchemaWriteState } from './write-okf-schemas.ts'; let tmpDir: string; diff --git a/packages/server/src/logger.test.ts b/packages/server/src/logger.test.ts index eb7378d25..a8f70fc90 100644 --- a/packages/server/src/logger.test.ts +++ b/packages/server/src/logger.test.ts @@ -1,6 +1,6 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { context, trace } from '@opentelemetry/api'; import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'; @@ -107,12 +107,18 @@ describe('Logger', () => { }); describe('PinoFileSink wiring', () => { + let fixtureDir: string; let tmp: string; beforeEach(() => { - tmp = mkdtempSync(join(tmpdir(), 'ok-logger-test-')); + fixtureDir = mkdtempSync(join(tmpdir(), 'ok-logger-test-')); + tmp = join(fixtureDir, 'project'); + mkdirSync(tmp); + vi.stubEnv('HOME', join(fixtureDir, 'home')); + vi.stubEnv('USERPROFILE', join(fixtureDir, 'home')); }); afterEach(() => { - if (tmp) rmSync(tmp, { recursive: true, force: true }); + vi.unstubAllEnvs(); + if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true }); }); interface ParsedLogRecord { @@ -172,13 +178,14 @@ describe('Logger', () => { options: { level: 'info' }, fileSink: { projectDir: tmp, maxBytes: 1_000_000 }, }); - const logsDir = join(tmp, '.ok', 'local', 'logs'); + const logsDir = dirname(logsCurrentPath(tmp)); expect(existsSync(logsDir)).toBe(false); logger.info({}, 'first-record'); await logger.flushFileSink(); expect(existsSync(logsDir)).toBe(true); + if (process.platform !== 'win32') expect(statSync(logsDir).mode & 0o777).toBe(0o700); expect(existsSync(logsCurrentPath(tmp))).toBe(true); }); @@ -304,11 +311,11 @@ describe('Logger', () => { expect(existsSync(logsCurrentPath(tmp))).toBe(false); }); - it('omits file destination when fileSink config is absent (no .ok/local/logs created)', async () => { + it('omits file destination when fileSink config is absent', async () => { const logger = new PinoLogger('test', { options: { level: 'info' }, }); - const logsDir = join(tmp, '.ok', 'local', 'logs'); + const logsDir = dirname(logsCurrentPath(tmp)); logger.info({}, 'no-file-sink-here'); await logger.flushFileSink(); diff --git a/packages/server/src/mcp-dual-transport-rig.test-helper.ts b/packages/server/src/mcp-dual-transport-rig.test-helper.ts index 3b9b73069..ae98071d5 100644 --- a/packages/server/src/mcp-dual-transport-rig.test-helper.ts +++ b/packages/server/src/mcp-dual-transport-rig.test-helper.ts @@ -5,7 +5,7 @@ import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; import type { Client } from '@modelcontextprotocol/sdk/client/index.js'; -import { type BootedServer, bootServer } from './boot.ts'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { ConfigSchema } from './config/schema.ts'; import { connectMcpTestClient } from './mcp/client.test-helper.ts'; import { createMcpHttpHandler, type McpHttpHandler } from './mcp-http.ts'; diff --git a/packages/server/src/mcp-lint-http-preservation.test.ts b/packages/server/src/mcp-lint-http-preservation.test.ts index b8c42a541..35e464c5f 100644 --- a/packages/server/src/mcp-lint-http-preservation.test.ts +++ b/packages/server/src/mcp-lint-http-preservation.test.ts @@ -7,7 +7,7 @@ import { OK_DIR } from '@inkeep/open-knowledge-core'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest'; -import { type BootedServer, bootServer } from './boot.ts'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { ConfigSchema } from './config/schema.ts'; import { __formatContributorsForTests, diff --git a/packages/server/src/nested-project-authority.test.ts b/packages/server/src/nested-project-authority.test.ts index d6e114c68..8ee444515 100644 --- a/packages/server/src/nested-project-authority.test.ts +++ b/packages/server/src/nested-project-authority.test.ts @@ -15,10 +15,10 @@ import * as encoding from 'lib0/encoding'; import { expect, test, vi } from 'vitest'; import WebSocket from 'ws'; import * as Y from 'yjs'; +import { type BootedServer, bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { testAuthorityRegistryPath } from '../../../test-support/server-authority-registry.test-helper.ts'; import * as assetWalk from './asset-walk.ts'; import { BacklinkIndex } from './backlink-index.ts'; -import { type BootedServer, bootServer } from './boot.ts'; import { bootCompositionRig } from './composition-rig.test-helper.ts'; import { ConfigSchema } from './config/schema.ts'; import * as contentFilterModule from './content-filter.ts'; diff --git a/packages/server/src/polled-path-watcher.test.ts b/packages/server/src/polled-path-watcher.test.ts index ebba98c67..d24ef478a 100644 --- a/packages/server/src/polled-path-watcher.test.ts +++ b/packages/server/src/polled-path-watcher.test.ts @@ -2,7 +2,7 @@ import { spawn } from 'node:child_process'; import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { logsCurrentPath } from './telemetry-file-sink.ts'; const PRIMITIVE = new URL('./polled-path-watcher.ts', import.meta.url).href; @@ -213,8 +213,11 @@ let dir: string; beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'ok-polled-watcher-')); + vi.stubEnv('HOME', join(dir, 'home')); + vi.stubEnv('USERPROFILE', join(dir, 'home')); }); afterEach(() => { + vi.unstubAllEnvs(); rmSync(dir, { recursive: true, force: true }); }); diff --git a/packages/server/src/project-logs-retention.test.ts b/packages/server/src/project-logs-retention.test.ts new file mode 100644 index 000000000..9593d0e08 --- /dev/null +++ b/packages/server/src/project-logs-retention.test.ts @@ -0,0 +1,484 @@ +import { randomUUID } from 'node:crypto'; +import * as nodeFs from 'node:fs'; +import { + closeSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, + writeSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { SERVER_AUTHORITY_LEASES_DIRNAME, SPAWN_ERROR_LOG } from '@inkeep/open-knowledge-core'; +import { okProjectLogsDir, openSpawnErrorLog } from '@inkeep/open-knowledge-core/server'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import * as fsTraced from './fs-traced.ts'; +import { getLogger } from './logger.ts'; +import { prepareProjectLogsDir, prepareSpawnProjectLogsDir } from './project-logs-retention.ts'; +import * as authority from './server-authority.ts'; +import { acquireServerAuthority, ServerAuthorityRegistryError } from './server-authority.ts'; +import * as authorityLeases from './server-authority-lease.ts'; +import { canonicalContentPath } from './server-content-policy.ts'; + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), +})); + +let root: string; +let home: string; +let registryPath: string; +const releases: Array<() => void> = []; +const eightDaysAgo = (): Date => new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + +beforeEach(() => { + root = realpathSync.native(mkdtempSync(join(tmpdir(), 'ok-project-log-retention-'))); + home = join(root, 'home'); + registryPath = join(root, 'authority', 'registry.sqlite'); + const projectDir = project('registry-init'); + const handle = acquireServerAuthority({ + scope: { kind: 'tree', path: projectDir, excluded: [] }, + projectDir, + serverInstanceId: randomUUID(), + registryPath, + }); + handle.release(); +}); + +afterEach(() => { + vi.restoreAllMocks(); + for (const release of releases.splice(0)) release(); + rmSync(root, { recursive: true, force: true }); +}); + +function project(name: string): string { + const path = join(root, name); + mkdirSync(path, { recursive: true }); + return path; +} + +function oldNamespace(projectDir: string): string { + const path = okProjectLogsDir(projectDir, home); + mkdirSync(path, { recursive: true }); + const file = join(path, 'server-current.jsonl'); + writeFileSync(file, 'retained diagnostic\n'); + const old = eightDaysAgo(); + utimesSync(file, old, old); + utimesSync(path, old, old); + return path; +} + +function prepare(projectDir = project('current')): string { + return prepareProjectLogsDir(projectDir, { home, authorityRegistryPath: registryPath }); +} + +function observeRegistryLockDuringTouch(path: string): boolean[] { + const observations: boolean[] = []; + const original = fsTraced.tracedUtimesSync; + vi.spyOn(fsTraced, 'tracedUtimesSync').mockImplementation((...args) => { + const contender = new DatabaseSync(path, { timeout: 0 }); + try { + try { + contender.exec('BEGIN IMMEDIATE'); + observations.push(false); + } catch (error) { + const code = (error as { errcode?: number }).errcode; + observations.push(typeof code === 'number' && ((code & 0xff) === 5 || (code & 0xff) === 6)); + } + } finally { + contender.close(); + } + return original(...args); + }); + return observations; +} + +test('reclaims old inactive namespaces after their project is removed and keeps recent evidence', () => { + const removed = project('removed'); + const old = oldNamespace(removed); + rmSync(removed, { recursive: true }); + const recent = oldNamespace(project('recent')); + writeFileSync(join(recent, SPAWN_ERROR_LOG), 'recent failure\n'); + prepare(); + expect(existsSync(old)).toBe(false); + expect(readFileSync(join(recent, SPAWN_ERROR_LOG), 'utf8')).toBe('recent failure\n'); +}); + +test('preserves arbitrarily quiet active authority even when every log timestamp is old', () => { + const projectDir = project('quiet'); + const path = oldNamespace(projectDir); + const handle = acquireServerAuthority({ + scope: { kind: 'tree', path: projectDir, excluded: [] }, + projectDir, + serverInstanceId: randomUUID(), + registryPath, + }); + releases.push(handle.release); + prepare(); + expect(readFileSync(join(path, 'server-current.jsonl'), 'utf8')).toBe('retained diagnostic\n'); + handle.release(); + prepare(); + expect(existsSync(path)).toBe(false); +}); + +test('a held canonical authority protects logs after its aliased ephemeral project is removed', () => { + const realParent = project('real-parent'); + const aliasParent = join(root, 'aliased-parent'); + symlinkSync(realParent, aliasParent, process.platform === 'win32' ? 'junction' : 'dir'); + const logicalProject = join(aliasParent, 'ephemeral-project'); + mkdirSync(logicalProject); + const path = oldNamespace(logicalProject); + const canonicalProject = canonicalContentPath(logicalProject); + const handle = acquireServerAuthority({ + scope: { kind: 'tree', path: canonicalProject, excluded: [] }, + projectDir: canonicalProject, + serverInstanceId: randomUUID(), + registryPath, + }); + releases.push(handle.release); + rmSync(logicalProject, { recursive: true }); + expect(okProjectLogsDir(logicalProject, home)).not.toBe(path); + prepare(); + expect(readFileSync(join(path, 'server-current.jsonl'), 'utf8')).toBe('retained diagnostic\n'); + handle.release(); + prepare(); + expect(existsSync(path)).toBe(false); +}); + +test.each(['sqlite-lifetime-lease:v1', 'future-protocol:v2'])( + 'an unverifiable lease (%s) prepares under the transaction but skips cleanup', + (protocol) => { + const path = oldNamespace(project('stale')); + const database = new DatabaseSync(registryPath); + database + .prepare('INSERT INTO server_authority VALUES (?, ?, ?, ?, ?, ?, ?)') + .run( + randomUUID(), + randomUUID(), + process.pid, + protocol, + project('unverifiable'), + JSON.stringify({ kind: 'tree', path: root, excluded: [] }), + new Date().toISOString(), + ); + database.close(); + const observations = observeRegistryLockDuringTouch(registryPath); + const current = prepare(); + expect(observations).toEqual([true]); + expect(existsSync(path)).toBe(true); + const fd = openSpawnErrorLog(join(current, SPAWN_ERROR_LOG), process.pid); + try { + writeSync(fd, 'new diagnostic survives\n'); + } finally { + closeSync(fd); + } + expect(readFileSync(join(current, SPAWN_ERROR_LOG), 'utf8')).toContain( + 'new diagnostic survives', + ); + }, +); + +test('an unlocked persisted lifetime lease permits old orphan namespace reclamation', () => { + const projectDir = project('crashed'); + const path = oldNamespace(projectDir); + const token = randomUUID(); + const leaseDir = join(root, 'authority', SERVER_AUTHORITY_LEASES_DIRNAME); + mkdirSync(leaseDir, { recursive: true }); + const lease = new DatabaseSync(join(leaseDir, `${token}.sqlite`)); + lease.exec('CREATE TABLE lease (owner_token TEXT PRIMARY KEY) STRICT'); + lease.prepare('INSERT INTO lease VALUES (?)').run(token); + lease.close(); + const registry = new DatabaseSync(registryPath); + registry + .prepare('INSERT INTO server_authority VALUES (?, ?, ?, ?, ?, ?, ?)') + .run( + randomUUID(), + token, + process.pid, + 'sqlite-lifetime-lease:v1', + projectDir, + JSON.stringify({ kind: 'tree', path: projectDir, excluded: [] }), + new Date().toISOString(), + ); + registry.close(); + prepare(); + expect(existsSync(path)).toBe(false); +}); + +test('an ownership lease access error skips cleanup without dropping new logging', () => { + const old = oldNamespace(project('old-access-denied')); + const holderProject = project('holder-access-denied'); + const handle = acquireServerAuthority({ + scope: { kind: 'tree', path: holderProject, excluded: [] }, + projectDir: holderProject, + serverInstanceId: randomUUID(), + registryPath, + }); + releases.push(handle.release); + vi.spyOn(authorityLeases, 'isServerAuthorityLeaseReleased').mockImplementationOnce(() => { + throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + }); + const observations = observeRegistryLockDuringTouch(registryPath); + const current = prepare(); + expect(observations).toEqual([true]); + expect(existsSync(old)).toBe(true); + writeFileSync(join(current, 'server-current.jsonl'), 'still logging\n'); + expect(readFileSync(join(current, 'server-current.jsonl'), 'utf8')).toBe('still logging\n'); +}); + +test('an unreadable registry aborts preparation without touching an unguarded namespace', () => { + const old = oldNamespace(project('old-registry-access-denied')); + const projectDir = project('new-registry-access-denied'); + const current = okProjectLogsDir(projectDir, home); + vi.spyOn(DatabaseSync.prototype, 'prepare').mockImplementationOnce(() => { + throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + }); + expect(() => prepare(projectDir)).toThrow(ServerAuthorityRegistryError); + expect(existsSync(old)).toBe(true); + expect(existsSync(current)).toBe(false); +}); + +test.each([5, 6, 261])( + 'ownership contention (%i) aborts preparation without an unguarded fallback', + (errcode) => { + const old = oldNamespace(project('old-contended')); + const projectDir = project('new-contended'); + const current = okProjectLogsDir(projectDir, home); + vi.spyOn(DatabaseSync.prototype, 'prepare').mockImplementationOnce(() => { + throw Object.assign(new Error('Database is busy'), { errcode }); + }); + expect(() => prepare(projectDir)).toThrow(ServerAuthorityRegistryError); + expect(existsSync(current)).toBe(false); + expect(existsSync(old)).toBe(true); + }, +); + +test('a missing registry is initialized before namespace refresh and pruning', () => { + const old = oldNamespace(project('old')); + const missingRegistry = join(root, 'missing', 'registry.sqlite'); + const observations = observeRegistryLockDuringTouch(missingRegistry); + const current = prepareProjectLogsDir(project('new'), { + home, + authorityRegistryPath: missingRegistry, + }); + expect(observations).toEqual([true]); + expect(existsSync(old)).toBe(false); + const registry = new DatabaseSync(missingRegistry, { readOnly: true }); + try { + expect(registry.prepare('PRAGMA user_version').get()?.user_version).toBe(2); + expect(registry.prepare('SELECT count(*) AS count FROM server_authority').get()?.count).toBe(0); + } finally { + registry.close(); + } + writeFileSync(join(current, 'server-current.jsonl'), 'still logging\n'); + expect(readFileSync(join(current, 'server-current.jsonl'), 'utf8')).toBe('still logging\n'); +}); + +test('an unavailable registry location aborts without an unguarded preparation', () => { + const old = oldNamespace(project('old-location-unavailable')); + const blocked = join(root, 'not-a-directory'); + writeFileSync(blocked, 'ordinary fixture file'); + const projectDir = project('new-location-unavailable'); + const current = okProjectLogsDir(projectDir, home); + expect(() => + prepareProjectLogsDir(projectDir, { + home, + authorityRegistryPath: join(blocked, 'registry.sqlite'), + }), + ).toThrow(ServerAuthorityRegistryError); + expect(existsSync(old)).toBe(true); + expect(existsSync(current)).toBe(false); +}); + +test('an optional namespace preparation failure leaves boot logging best-effort and skips pruning', () => { + const old = oldNamespace(project('old-preparation-failed')); + const projectDir = project('preparation-failed'); + const current = okProjectLogsDir(projectDir, home); + const original = fsTraced.tracedMkdirSync; + vi.spyOn(fsTraced, 'tracedMkdirSync').mockImplementation((path, options) => { + if (path === current) throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + return original(path, options); + }); + expect(prepare(projectDir)).toBe(current); + expect(existsSync(old)).toBe(true); + expect(existsSync(current)).toBe(false); +}); + +test('a namespace pruning error stays best-effort rather than becoming an authority failure', () => { + const old = oldNamespace(project('old-pruning-failed')); + vi.spyOn(fsTraced, 'tracedRmSync').mockImplementationOnce(() => { + throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + }); + const current = prepare(); + expect(existsSync(old)).toBe(true); + writeFileSync(join(current, 'server-current.jsonl'), 'logging continues\n'); + expect(readFileSync(join(current, 'server-current.jsonl'), 'utf8')).toBe('logging continues\n'); +}); + +test('preparing a new attempt refreshes its namespace under the real SQLite transaction', () => { + const projectDir = project('attempt'); + const path = oldNamespace(projectDir); + const observations = observeRegistryLockDuringTouch(registryPath); + expect(prepare(projectDir)).toBe(path); + expect(observations).toEqual([true]); + expect(Date.now() - statSync(path).mtimeMs).toBeLessThan(60_000); + vi.restoreAllMocks(); + prepare(project('other')); + const fd = openSpawnErrorLog(join(path, SPAWN_ERROR_LOG), process.pid); + try { + writeSync(fd, 'late startup stderr\n'); + } finally { + closeSync(fd); + } + expect(readFileSync(join(path, SPAWN_ERROR_LOG), 'utf8')).toContain('late startup stderr'); +}); + +test('preserves symlinked namespaces and unknown contents rather than following or removing them', () => { + const target = join(root, 'external-diagnostics'); + mkdirSync(target, { recursive: true }); + writeFileSync(join(target, 'server-current.jsonl'), 'external diagnostic\n'); + const aliasProject = project('alias'); + const alias = okProjectLogsDir(aliasProject, home); + mkdirSync(join(alias, '..'), { recursive: true }); + symlinkSync(target, alias, process.platform === 'win32' ? 'junction' : 'dir'); + const unknown = oldNamespace(project('unknown')); + writeFileSync(join(unknown, 'not-an-ok-diagnostic.txt'), 'do not remove'); + const old = eightDaysAgo(); + utimesSync(unknown, old, old); + utimesSync(join(target, 'server-current.jsonl'), old, old); + utimesSync(target, old, old); + prepare(); + expect(lstatSync(alias).isSymbolicLink()).toBe(true); + expect(readFileSync(join(alias, 'server-current.jsonl'), 'utf8')).toBe('external diagnostic\n'); + expect(readFileSync(join(unknown, 'not-an-ok-diagnostic.txt'), 'utf8')).toBe('do not remove'); +}); + +test('spawn preparation lets the event loop release a contended registry without sweeping', async () => { + const old = oldNamespace(project('inactive-before-spawn')); + const contender = new DatabaseSync(registryPath); + contender.exec('BEGIN IMMEDIATE'); + const release = new Promise((resolve) => { + setImmediate(() => { + contender.close(); + resolve(); + }); + }); + const projectDir = project('spawn-attempt'); + const result = prepareSpawnProjectLogsDir(projectDir, { + home, + authorityRegistryPath: registryPath, + }).then( + (path) => ({ path }), + (error: unknown) => ({ error }), + ); + await release; + expect(await result).toEqual({ path: okProjectLogsDir(projectDir, home) }); + expect(existsSync(old)).toBe(true); +}); + +test('namespace enumeration does not hold the ownership write transaction', () => { + const old = oldNamespace(project('scan-outside-transaction')); + const logRoot = dirname(old); + const observations: boolean[] = []; + const original = nodeFs.readdirSync; + vi.spyOn(nodeFs, 'readdirSync').mockImplementation((...args: Parameters) => { + if (args[0] === logRoot) { + const contender = new DatabaseSync(registryPath, { timeout: 0 }); + try { + try { + contender.exec('BEGIN IMMEDIATE'); + observations.push(false); + } catch { + observations.push(true); + } + } finally { + contender.close(); + } + } + return original(...args); + }); + prepare(); + expect(observations).toEqual([false]); + expect(existsSync(old)).toBe(false); +}); + +test.each(['contention', 'foreign-lease'])( + 'cleanup stops with one warning when ownership changes after enumeration (%s)', + (failure) => { + const first = oldNamespace(project('contended-first')); + const second = oldNamespace(project('contended-second')); + const logRoot = dirname(first); + const original = nodeFs.readdirSync; + let contender: DatabaseSync | undefined; + const warn = vi.spyOn(getLogger('project-logs-retention'), 'warn'); + const authorityWarn = vi.spyOn(getLogger('server-authority'), 'warn'); + vi.spyOn(nodeFs, 'readdirSync').mockImplementation((...args: Parameters) => { + if (args[0] === logRoot && !contender) { + contender = new DatabaseSync(registryPath, { timeout: 0 }); + if (failure === 'contention') { + contender.exec('BEGIN IMMEDIATE'); + } else { + contender + .prepare('INSERT INTO server_authority VALUES (?, ?, ?, ?, ?, ?, ?)') + .run( + randomUUID(), + randomUUID(), + process.pid, + 'future-protocol:v2', + root, + JSON.stringify({ kind: 'tree', path: root, excluded: [] }), + new Date().toISOString(), + ); + } + } + return original(...args); + }); + try { + prepare(); + expect(warn.mock.calls.length + authorityWarn.mock.calls.length).toBe(1); + expect(existsSync(first)).toBe(true); + expect(existsSync(second)).toBe(true); + } finally { + contender?.close(); + } + }, +); + +test.each(['freshness', 'unknown-file'])( + 'the deletion transaction rechecks changed %s after candidate selection', + (change) => { + const old = oldNamespace(project('changed-after-selection')); + const original = authority.withServerAuthorityProjectDirs; + let calls = 0; + vi.spyOn(authority, 'withServerAuthorityProjectDirs').mockImplementation( + (fn, path, timeout) => { + calls += 1; + if (calls === 2) { + if (change === 'freshness') { + const now = new Date(); + utimesSync(old, now, now); + } else { + writeFileSync(join(old, 'unfamiliar.txt'), 'preserve this file'); + const aged = eightDaysAgo(); + utimesSync(old, aged, aged); + } + } + return original(fn, path, timeout); + }, + ); + prepare(); + expect(calls).toBe(2); + expect(existsSync(old)).toBe(true); + expect(readFileSync(join(old, 'server-current.jsonl'), 'utf8')).toBe('retained diagnostic\n'); + if (change === 'unknown-file') + expect(readFileSync(join(old, 'unfamiliar.txt'), 'utf8')).toBe('preserve this file'); + }, +); diff --git a/packages/server/src/project-logs-retention.ts b/packages/server/src/project-logs-retention.ts new file mode 100644 index 000000000..821a39a5c --- /dev/null +++ b/packages/server/src/project-logs-retention.ts @@ -0,0 +1,132 @@ +import { lstatSync, readdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { PROJECT_DIAGNOSTIC_FILES } from '@inkeep/open-knowledge-core'; +import { + isProjectLogsKey, + okProjectLogsDir, + projectLogsKeyForCanonicalPath, +} from '@inkeep/open-knowledge-core/server'; +import { tracedMkdirSync, tracedRmSync, tracedUtimesSync } from './fs-traced.ts'; +import { getLogger } from './logger.ts'; +import { + ServerAuthorityRegistryError, + withServerAuthorityProjectDirs, + withServerAuthorityRegistryTransaction, +} from './server-authority.ts'; + +const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; +const PROJECT_LOG_FILES: ReadonlySet = new Set(Object.values(PROJECT_DIAGNOSTIC_FILES)); + +function expiredNamespace(path: string, cutoff: number): boolean { + const directory = lstatSync(path); + if (!directory.isDirectory() || directory.isSymbolicLink() || directory.mtimeMs >= cutoff) + return false; + for (const entry of readdirSync(path, { withFileTypes: true })) { + if (!entry.isFile() || !PROJECT_LOG_FILES.has(entry.name)) return false; + const file = lstatSync(join(path, entry.name)); + if (!file.isFile() || file.isSymbolicLink() || file.mtimeMs >= cutoff) return false; + } + return true; +} + +function refreshNamespace(path: string): boolean { + tracedMkdirSync(path, { recursive: true, mode: 0o700 }); + const directory = lstatSync(path); + if (!directory.isDirectory() || directory.isSymbolicLink()) return false; + const now = new Date(); + tracedUtimesSync(path, now, now); + return true; +} + +function pruneProjectLogs(root: string, authorityRegistryPath?: string): void { + const directory = lstatSync(root); + if (!directory.isDirectory() || directory.isSymbolicLink()) return; + const cutoff = Date.now() - MAX_AGE_MS; + for (const entry of readdirSync(root, { withFileTypes: true })) { + if (!entry.isDirectory() || !isProjectLogsKey(entry.name)) continue; + const path = join(root, entry.name); + try { + if (!expiredNamespace(path, cutoff)) continue; + let verified = true; + withServerAuthorityProjectDirs( + (activeProjectDirs) => { + if (activeProjectDirs === null) { + verified = false; + return; + } + for (const projectDir of activeProjectDirs) { + if (projectLogsKeyForCanonicalPath(projectDir) === entry.name) return; + } + if (expiredNamespace(path, cutoff)) tracedRmSync(path, { recursive: true }); + }, + authorityRegistryPath, + 0, + ); + if (!verified) return; + } catch (err) { + getLogger('project-logs-retention').warn( + { err }, + 'Inactive project diagnostic cleanup skipped', + ); + if (err instanceof ServerAuthorityRegistryError) return; + } + } +} + +export function prepareProjectLogsDir( + projectDir: string, + options: { home?: string; authorityRegistryPath?: string } = {}, +): string { + const path = okProjectLogsDir(projectDir, options.home); + let canPrune = false; + withServerAuthorityProjectDirs((activeProjectDirs) => { + try { + canPrune = refreshNamespace(path) && activeProjectDirs !== null; + } catch (err) { + getLogger('project-logs-retention').warn( + { err }, + 'Project diagnostic namespace could not be refreshed', + ); + } + }, options.authorityRegistryPath); + if (canPrune) { + try { + pruneProjectLogs(dirname(path), options.authorityRegistryPath); + } catch (err) { + getLogger('project-logs-retention').warn( + { err }, + 'Project diagnostic cleanup could not scan namespaces', + ); + } + } + return path; +} + +export async function prepareSpawnProjectLogsDir( + projectDir: string, + options: { home?: string; authorityRegistryPath?: string } = {}, +): Promise { + const path = okProjectLogsDir(projectDir, options.home); + const deadline = Date.now() + 5_000; + for (;;) { + try { + withServerAuthorityRegistryTransaction( + () => { + refreshNamespace(path); + }, + options.authorityRegistryPath, + 0, + ); + return path; + } catch (err) { + if ( + !(err instanceof ServerAuthorityRegistryError) || + err.kind !== 'contention' || + Date.now() >= deadline + ) + throw err; + await delay(25); + } + } +} diff --git a/packages/server/src/server-authority.test.ts b/packages/server/src/server-authority.test.ts index 11369b1ed..c3f2f576c 100644 --- a/packages/server/src/server-authority.test.ts +++ b/packages/server/src/server-authority.test.ts @@ -6,9 +6,11 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import { fileURLToPath } from 'node:url'; +import { okProjectLogsDir } from '@inkeep/open-knowledge-core/server'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { z } from 'zod'; import { withHiddenWindowsConsole } from './child-process-windows-hide.ts'; +import { prepareProjectLogsDir } from './project-logs-retention.ts'; import { acquireServerAuthority, type ContentScope, @@ -668,3 +670,37 @@ describe('acquireServerAuthority', () => { expect(() => claim(tree())).toThrow(ServerAuthorityCollisionError); }); }); + +test('retention preserves an old quiet independent holder until the held child actually exits', async () => { + const worker = await startWorker(tree()); + expect((await worker.send('acquire')).status).toBe('acquired'); + const registry = new DatabaseSync(registryPath, { readOnly: true }); + let projectDir: string; + try { + const value = registry.prepare('SELECT project_dir FROM server_authority').get()?.project_dir; + if (typeof value !== 'string') throw new Error('Worker project directory absent'); + projectDir = value; + } finally { + registry.close(); + } + const home = join(directory, 'retention-home'); + const active = okProjectLogsDir(projectDir, home); + const inactive = okProjectLogsDir(join(directory, 'inactive-project'), home); + const old = new Date(Date.now() - 8 * 24 * 60 * 60 * 1000); + for (const path of [active, inactive]) { + mkdirSync(path, { recursive: true }); + const file = join(path, 'server-current.jsonl'); + writeFileSync(file, 'quiet diagnostic\n'); + fs.utimesSync(file, old, old); + fs.utimesSync(path, old, old); + } + const current = join(directory, 'current-project'); + mkdirSync(current); + prepareProjectLogsDir(current, { home, authorityRegistryPath: registryPath }); + expect(fs.existsSync(inactive)).toBe(false); + expect(fs.readFileSync(join(active, 'server-current.jsonl'), 'utf8')).toBe('quiet diagnostic\n'); + worker.child.kill('SIGKILL'); + await worker.exited; + prepareProjectLogsDir(current, { home, authorityRegistryPath: registryPath }); + expect(fs.existsSync(active)).toBe(false); +}); diff --git a/packages/server/src/server-authority.ts b/packages/server/src/server-authority.ts index a6b8b89f7..1c41f0589 100644 --- a/packages/server/src/server-authority.ts +++ b/packages/server/src/server-authority.ts @@ -210,7 +210,11 @@ function immutableScope(value: unknown): ContentScope { : Object.freeze({ ...scope, excluded: Object.freeze(scope.excluded) }); } -function authoritySpan(operation: 'acquire' | 'release', registryPath: string, fn: () => T): T { +function authoritySpan( + operation: 'acquire' | 'release' | 'retention', + registryPath: string, + fn: () => T, +): T { return withSpanSync( 'fs.serverAuthority', { @@ -225,6 +229,160 @@ function authoritySpan(operation: 'acquire' | 'release', registryPath: string ); } +function resolveServerAuthorityRegistryPath(override?: string): string { + let osHome: string; + try { + osHome = userInfo().homedir; + } catch (error) { + if (override === undefined) { + throw new ServerAuthorityRegistryError('unavailable', undefined, error); + } + osHome = dirname(override); + } + return pathSchema.parse( + override ?? resolve(osHome, OK_DIR, LOCAL_DIR, SERVER_AUTHORITY_REGISTRY_FILENAME), + ); +} + +function openServerAuthorityRegistry(registryPath: string, timeout = 5_000): DatabaseSync { + let database: DatabaseSync | undefined; + try { + tracedMkdirSync(dirname(registryPath), { recursive: true, mode: 0o700 }); + try { + tracedWriteFileSync(registryPath, '', { flag: 'wx', mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + tracedChmodSync(registryPath, 0o600); + database = new DatabaseSync(registryPath, { timeout }); + database.exec('BEGIN IMMEDIATE'); + const version = database.prepare('PRAGMA user_version').get()?.user_version; + const emptyLegacy = + version === 1 && + database + .prepare('PRAGMA table_info(server_authority)') + .all() + .map((column) => column.name) + .join(',') === + 'server_instance_id,owner_token,pid,hostname,machine_id,project_dir,scope_json,started_at' && + database.prepare('SELECT count(*) AS count FROM server_authority').get()?.count === 0; + if (version !== 0 && version !== 2 && !emptyLegacy) { + throw new ServerAuthorityRegistryError( + version === 1 ? 'retired-version' : 'unsupported-version', + registryPath, + new Error(`Registry layout version ${version} is not supported by this runtime`), + ); + } + if (database.prepare('PRAGMA journal_mode').get()?.journal_mode !== 'delete') + throw new ServerAuthorityRegistryError( + 'unsupported-version', + registryPath, + new Error('Content ownership requires SQLite DELETE-mode file locking'), + ); + if (emptyLegacy) database.exec('DROP TABLE server_authority'); + database.exec(` + CREATE TABLE IF NOT EXISTS server_authority ( + server_instance_id TEXT PRIMARY KEY, + owner_token TEXT NOT NULL, + pid INTEGER NOT NULL, + lease_protocol TEXT NOT NULL, + project_dir TEXT NOT NULL, + scope_json TEXT NOT NULL, + started_at TEXT NOT NULL + ) STRICT; + PRAGMA user_version = 2; + `); + return database; + } catch (error) { + database?.close(); + if (error instanceof ServerAuthorityRegistryError) throw error; + throw new ServerAuthorityRegistryError( + classifyServerAuthorityRegistryError(error), + registryPath, + error, + ); + } +} + +export function withServerAuthorityRegistryTransaction( + fn: (database: DatabaseSync, registryPath: string) => void, + registryPathOverride?: string, + timeout = 5_000, +): void { + const registryPath = resolveServerAuthorityRegistryPath(registryPathOverride); + authoritySpan('retention', registryPath, () => { + const database = openServerAuthorityRegistry(registryPath, timeout); + try { + fn(database, registryPath); + try { + database.exec('COMMIT'); + } catch (error) { + throw new ServerAuthorityRegistryError( + classifyServerAuthorityRegistryError(error), + registryPath, + error, + ); + } + } finally { + database.close(); + } + }); +} + +export function withServerAuthorityProjectDirs( + fn: (projectDirs: ReadonlySet | null) => void, + registryPathOverride?: string, + timeout = 5_000, +): void { + withServerAuthorityRegistryTransaction( + (database, registryPath) => { + let holders: Array>; + try { + holders = database + .prepare('SELECT * FROM server_authority') + .all() + .map((value) => { + const row = holderRowSchema.parse(value); + immutableScope(JSON.parse(row.scope_json)); + return row; + }); + } catch (error) { + throw new ServerAuthorityRegistryError( + classifyServerAuthorityRegistryError(error), + registryPath, + error, + ); + } + const active = new Set(); + let verified = true; + for (const row of holders) { + try { + if (row.lease_protocol !== SERVER_AUTHORITY_LEASE_PROTOCOL) { + getLogger('server-authority').warn( + {}, + 'Diagnostic cleanup skipped: unknown ownership lease protocol', + ); + verified = false; + break; + } + if (!isServerAuthorityLeaseReleased(registryPath, row.owner_token)) + active.add(row.project_dir); + } catch (err) { + getLogger('server-authority').warn( + { err }, + 'Diagnostic cleanup skipped: ownership lease could not be verified', + ); + verified = false; + break; + } + } + fn(verified ? active : null); + }, + registryPathOverride, + timeout, + ); +} + export function acquireServerAuthority(options: { scope: ContentScope; projectDir: string; @@ -234,18 +392,7 @@ export function acquireServerAuthority(options: { const scope = immutableScope(options.scope); const projectDir = pathSchema.parse(options.projectDir); const serverInstanceId = z.string().min(1).parse(options.serverInstanceId); - let osHome: string; - try { - osHome = userInfo().homedir; - } catch (error) { - if (options.registryPath === undefined) { - throw new ServerAuthorityRegistryError('unavailable', undefined, error); - } - osHome = dirname(options.registryPath); - } - const registryPath = - options.registryPath ?? resolve(osHome, OK_DIR, LOCAL_DIR, SERVER_AUTHORITY_REGISTRY_FILENAME); - pathSchema.parse(registryPath); + const registryPath = resolveServerAuthorityRegistryPath(options.registryPath); const ownerToken = randomUUID(); let releaseLease: (() => void) | undefined; const db = authoritySpan('acquire', registryPath, () => { @@ -257,51 +404,7 @@ export function acquireServerAuthority(options: { startedAt: string; }> = []; try { - tracedMkdirSync(dirname(registryPath), { recursive: true, mode: 0o700 }); - try { - tracedWriteFileSync(registryPath, '', { flag: 'wx', mode: 0o600 }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - } - tracedChmodSync(registryPath, 0o600); - database = new DatabaseSync(registryPath, { timeout: 5_000 }); - database.exec('BEGIN IMMEDIATE'); - const version = database.prepare('PRAGMA user_version').get()?.user_version; - const emptyLegacy = - version === 1 && - database - .prepare('PRAGMA table_info(server_authority)') - .all() - .map((column) => column.name) - .join(',') === - 'server_instance_id,owner_token,pid,hostname,machine_id,project_dir,scope_json,started_at' && - database.prepare('SELECT count(*) AS count FROM server_authority').get()?.count === 0; - if (version !== 0 && version !== 2 && !emptyLegacy) { - throw new ServerAuthorityRegistryError( - version === 1 ? 'retired-version' : 'unsupported-version', - registryPath, - new Error(`Registry layout version ${version} is not supported by this runtime`), - ); - } - if (database.prepare('PRAGMA journal_mode').get()?.journal_mode !== 'delete') - throw new ServerAuthorityRegistryError( - 'unsupported-version', - registryPath, - new Error('Content ownership requires SQLite DELETE-mode file locking'), - ); - if (emptyLegacy) database.exec('DROP TABLE server_authority'); - database.exec(` - CREATE TABLE IF NOT EXISTS server_authority ( - server_instance_id TEXT PRIMARY KEY, - owner_token TEXT NOT NULL, - pid INTEGER NOT NULL, - lease_protocol TEXT NOT NULL, - project_dir TEXT NOT NULL, - scope_json TEXT NOT NULL, - started_at TEXT NOT NULL - ) STRICT; - PRAGMA user_version = 2; - `); + database = openServerAuthorityRegistry(registryPath); const holders = database .prepare('SELECT * FROM server_authority') .all() diff --git a/packages/server/src/server-factory.conflict-authority.test.ts b/packages/server/src/server-factory.conflict-authority.test.ts index 2dd20e55d..e62cd3070 100644 --- a/packages/server/src/server-factory.conflict-authority.test.ts +++ b/packages/server/src/server-factory.conflict-authority.test.ts @@ -13,8 +13,8 @@ import { } from '@inkeep/open-knowledge-core'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import * as Y from 'yjs'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts'; -import { bootServer } from './boot.ts'; import { ConfigSchema } from './config/schema.ts'; import * as headWatcher from './head-watcher.ts'; import type { ServerInstance } from './server-factory.ts'; diff --git a/packages/server/src/server-factory.conflict-ledger-lifecycle.test.ts b/packages/server/src/server-factory.conflict-ledger-lifecycle.test.ts index 12bac32af..a0e8b2d67 100644 --- a/packages/server/src/server-factory.conflict-ledger-lifecycle.test.ts +++ b/packages/server/src/server-factory.conflict-ledger-lifecycle.test.ts @@ -7,8 +7,8 @@ import { setTimeout as wait } from 'node:timers/promises'; import { promisify } from 'node:util'; import { OK_DIR } from '@inkeep/open-knowledge-core'; import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts'; -import { bootServer } from './boot.ts'; import { ConfigSchema } from './config/schema.ts'; const execFileAsync = promisify(execFile); diff --git a/packages/server/src/shutdown-progress.test.ts b/packages/server/src/shutdown-progress.test.ts index 71c054950..428172ab6 100644 --- a/packages/server/src/shutdown-progress.test.ts +++ b/packages/server/src/shutdown-progress.test.ts @@ -3,8 +3,8 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { expect, test, vi } from 'vitest'; +import { bootServer } from '../../../test-support/boot-server.test-helper.ts'; import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts'; -import { bootServer } from './boot.ts'; import { ConfigSchema } from './config/schema.ts'; import { getLogger } from './logger.ts'; import { ensureProjectGit } from './project-git.ts'; diff --git a/packages/server/src/telemetry-file-sink.test.ts b/packages/server/src/telemetry-file-sink.test.ts index 0ade3e37b..170037e14 100644 --- a/packages/server/src/telemetry-file-sink.test.ts +++ b/packages/server/src/telemetry-file-sink.test.ts @@ -8,10 +8,12 @@ import { writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { basename, dirname, join } from 'node:path'; +import { okUserHomeDir } from '@inkeep/open-knowledge-core/server'; import { context, trace } from '@opentelemetry/api'; import { BasicTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { afterEach, beforeEach, describe, expect, test } from 'vitest'; +import pino from 'pino'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { DEFAULT_MAX_VALUE_BYTES, FileSpanExporter, @@ -26,11 +28,17 @@ import { } from './telemetry-file-sink.ts'; let tmp: string; +let fixtureRoot: string; beforeEach(() => { - tmp = mkdtempSync(join(tmpdir(), 'ok-file-span-test-')); + fixtureRoot = mkdtempSync(join(tmpdir(), 'ok-file-span-test-')); + tmp = join(fixtureRoot, 'project'); + mkdirSync(tmp); + vi.stubEnv('HOME', join(fixtureRoot, 'home')); + vi.stubEnv('USERPROFILE', join(fixtureRoot, 'home')); }); afterEach(() => { - if (tmp) rmSync(tmp, { recursive: true, force: true }); + vi.unstubAllEnvs(); + if (fixtureRoot) rmSync(fixtureRoot, { recursive: true, force: true }); }); function makeProvider(exporter: FileSpanExporter): BasicTracerProvider { @@ -114,7 +122,7 @@ describe('FileSpanExporter', () => { test('parent directory is created lazily on first export', async () => { const exporter = new FileSpanExporter({ projectDir: tmp, maxBytes: 1_000_000 }); - const telemetryDir = join(tmp, '.ok', 'local', 'telemetry'); + const telemetryDir = dirname(spansCurrentPath(tmp)); expect(existsSync(telemetryDir)).toBe(false); const provider = makeProvider(exporter); @@ -122,6 +130,7 @@ describe('FileSpanExporter', () => { await provider.forceFlush(); expect(existsSync(telemetryDir)).toBe(true); + if (process.platform !== 'win32') expect(statSync(telemetryDir).mode & 0o777).toBe(0o700); expect(existsSync(spansCurrentPath(tmp))).toBe(true); await provider.shutdown(); @@ -265,7 +274,74 @@ describe('FileSpanExporter', () => { }); }); +describe('project diagnostics location', () => { + test('real Pino and span writes stay outside the project in the same user namespace', async () => { + const sink = new PinoFileSink({ projectDir: tmp, maxBytes: 1_000_000 }); + const provider = makeProvider(new FileSpanExporter({ projectDir: tmp, maxBytes: 1_000_000 })); + try { + pino({}, sink).info('project log canary'); + provider.getTracer('test').startSpan('project span canary').end(); + await Promise.all([sink.drain(), provider.forceFlush()]); + + const logsPath = logsCurrentPath(tmp); + const spansPath = spansCurrentPath(tmp); + expect(readFileSync(logsPath, 'utf8')).toContain('project log canary'); + expect(firstSpanName(parseEnvelope(readLines(spansPath)[0] ?? ''))).toBe( + 'project span canary', + ); + const namespace = dirname(logsPath); + expect(dirname(namespace)).toBe(join(okUserHomeDir(), 'logs', 'projects')); + expect(basename(namespace)).toMatch(/^[a-f0-9]{64}$/); + expect(dirname(spansPath)).toBe(namespace); + expect(existsSync(join(tmp, '.ok', 'local', 'logs'))).toBe(false); + expect(existsSync(join(tmp, '.ok', 'local', 'telemetry'))).toBe(false); + } finally { + await provider.shutdown(); + sink.destroy(); + } + }); + + test('two projects with the same name do not mix log records', async () => { + const otherProject = join(fixtureRoot, 'other', 'project'); + mkdirSync(otherProject, { recursive: true }); + const first = new PinoFileSink({ projectDir: tmp, maxBytes: 1_000_000 }); + const second = new PinoFileSink({ projectDir: otherProject, maxBytes: 1_000_000 }); + try { + pino({}, first).info('first project canary'); + pino({}, second).info('second project canary'); + await Promise.all([first.drain(), second.drain()]); + + const firstLog = readFileSync(logsCurrentPath(tmp), 'utf8'); + const secondLog = readFileSync(logsCurrentPath(otherProject), 'utf8'); + expect(firstLog).toContain('first project canary'); + expect(firstLog).not.toContain('second project canary'); + expect(secondLog).toContain('second project canary'); + expect(secondLog).not.toContain('first project canary'); + } finally { + first.destroy(); + second.destroy(); + } + }); +}); + describe('RotatingAppender (shared rotation primitive)', () => { + test('recreates a namespace removed while a retained appender was inactive', async () => { + const currentPath = logsCurrentPath(tmp); + const namespace = dirname(currentPath); + const appender = new RotatingAppender({ + currentPath, + previousPath: logsPreviousPath(tmp), + maxBytes: 1024, + }); + await appender.append('first\n'); + await appender.drain(); + rmSync(namespace, { recursive: true }); + await expect(appender.append('after reactivation\n')).resolves.toBeUndefined(); + await appender.drain(); + expect(readFileSync(currentPath, 'utf8')).toBe('after reactivation\n'); + if (process.platform !== 'win32') expect(statSync(namespace).mode & 0o777).toBe(0o700); + }); + test('appends bytes, creates parent dir lazily, and rotates over cap', async () => { const currentPath = join(tmp, 'logs', 'a-current.jsonl'); const previousPath = join(tmp, 'logs', 'a-prev.jsonl'); diff --git a/packages/server/src/telemetry-file-sink.ts b/packages/server/src/telemetry-file-sink.ts index d8470d71a..85bfb0938 100644 --- a/packages/server/src/telemetry-file-sink.ts +++ b/packages/server/src/telemetry-file-sink.ts @@ -2,6 +2,8 @@ import { statSync } from 'node:fs'; import { mkdir, rename, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import { Writable } from 'node:stream'; +import { PROJECT_DIAGNOSTIC_FILES } from '@inkeep/open-knowledge-core'; +import { okProjectLogsDir } from '@inkeep/open-knowledge-core/server'; import type { Context } from '@opentelemetry/api'; import type { ExportResult } from '@opentelemetry/core'; import { ExportResultCode } from '@opentelemetry/core'; @@ -60,10 +62,18 @@ export class RotatingAppender { async #doAppend(data: string | Uint8Array): Promise { if (!this.#state.parentDirEnsured) { - await mkdir(dirname(this.#currentPath), { recursive: true }); + await mkdir(dirname(this.#currentPath), { recursive: true, mode: 0o700 }); this.#state.parentDirEnsured = true; } - await writeFile(this.#currentPath, data, { flag: 'a' }); + try { + await writeFile(this.#currentPath, data, { flag: 'a' }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err; + this.#state.parentDirEnsured = false; + await mkdir(dirname(this.#currentPath), { recursive: true, mode: 0o700 }); + this.#state.parentDirEnsured = true; + await writeFile(this.#currentPath, data, { flag: 'a' }); + } let size: number; try { size = statSync(this.#currentPath).size; @@ -86,16 +96,12 @@ export interface FileSpanExporterOpts { maxBytes: number; } -const TELEMETRY_SUBDIR = ['.ok', 'local', 'telemetry'] as const; -const CURRENT_FILENAME = 'spans-current.jsonl'; -const PREVIOUS_FILENAME = 'spans-prev.jsonl'; - export function spansCurrentPath(projectDir: string): string { - return join(projectDir, ...TELEMETRY_SUBDIR, CURRENT_FILENAME); + return join(okProjectLogsDir(projectDir), PROJECT_DIAGNOSTIC_FILES.spansCurrent); } export function spansPreviousPath(projectDir: string): string { - return join(projectDir, ...TELEMETRY_SUBDIR, PREVIOUS_FILENAME); + return join(okProjectLogsDir(projectDir), PROJECT_DIAGNOSTIC_FILES.spansPrevious); } export class FileSpanExporter implements SpanExporter { @@ -235,16 +241,12 @@ export class ScrubbingSpanProcessor implements SpanProcessor { async shutdown(): Promise {} } -const LOGS_SUBDIR = ['.ok', 'local', 'logs'] as const; -const LOGS_CURRENT_FILENAME = 'server-current.jsonl'; -const LOGS_PREVIOUS_FILENAME = 'server-prev.jsonl'; - export function logsCurrentPath(projectDir: string): string { - return join(projectDir, ...LOGS_SUBDIR, LOGS_CURRENT_FILENAME); + return join(okProjectLogsDir(projectDir), PROJECT_DIAGNOSTIC_FILES.logsCurrent); } export function logsPreviousPath(projectDir: string): string { - return join(projectDir, ...LOGS_SUBDIR, LOGS_PREVIOUS_FILENAME); + return join(okProjectLogsDir(projectDir), PROJECT_DIAGNOSTIC_FILES.logsPrevious); } export interface PinoFileSinkOpts { diff --git a/packages/server/src/telemetry.test.ts b/packages/server/src/telemetry.test.ts index 4c86b6ed2..fc57353c1 100644 --- a/packages/server/src/telemetry.test.ts +++ b/packages/server/src/telemetry.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { context, metrics, propagation, trace } from '@opentelemetry/api'; @@ -7,7 +7,7 @@ import { InMemorySpanExporter, SimpleSpanProcessor, } from '@opentelemetry/sdk-trace-base'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { getMeter, getTracer, @@ -28,6 +28,7 @@ function makeLocalSinkOpts(projectDir: string): LocalSinkOptions { } describe('Telemetry', () => { + let fixtureDir: string; let tmp: string; beforeEach(async () => { await shutdownTelemetry(); @@ -36,7 +37,11 @@ describe('Telemetry', () => { context.disable(); delete process.env.OTEL_EXPORTER_OTLP_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_TIMEOUT; - tmp = mkdtempSync(join(tmpdir(), 'ok-telemetry-test-')); + fixtureDir = mkdtempSync(join(tmpdir(), 'ok-telemetry-test-')); + tmp = join(fixtureDir, 'project'); + mkdirSync(tmp); + vi.stubEnv('HOME', join(fixtureDir, 'home')); + vi.stubEnv('USERPROFILE', join(fixtureDir, 'home')); }); afterEach(async () => { @@ -44,7 +49,8 @@ describe('Telemetry', () => { trace.disable(); metrics.disable(); context.disable(); - if (tmp) rmSync(tmp, { recursive: true, force: true }); + vi.unstubAllEnvs(); + if (fixtureDir) rmSync(fixtureDir, { recursive: true, force: true }); }); describe('Gate combination 1 — both off (file sink absent + push disabled)', () => { diff --git a/test-support/boot-server.test-helper.ts b/test-support/boot-server.test-helper.ts new file mode 100644 index 000000000..0eb68e66a --- /dev/null +++ b/test-support/boot-server.test-helper.ts @@ -0,0 +1,11 @@ +import { type BootServerOptions, bootServer as boot } from '../packages/server/src/boot.ts'; +import { testAuthorityRegistryPath } from './server-authority-registry.test-helper.ts'; + +export type { BootedServer, BootServerOptions } from '../packages/server/src/boot.ts'; + +export function bootServer(options: BootServerOptions) { + return boot({ + ...options, + authorityRegistryPath: options.authorityRegistryPath ?? testAuthorityRegistryPath, + }); +} diff --git a/test-support/server-authority-registry.setup.ts b/test-support/server-authority-registry.setup.ts index d7255e084..747dd9841 100644 --- a/test-support/server-authority-registry.setup.ts +++ b/test-support/server-authority-registry.setup.ts @@ -1,4 +1,17 @@ -import { afterAll } from 'vitest'; -import { releaseTestAuthorityRegistry } from './server-authority-registry.test-helper.ts'; +import { mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { afterAll, beforeAll } from 'vitest'; +import { + releaseTestAuthorityRegistry, + testAuthorityRegistryPath, +} from './server-authority-registry.test-helper.ts'; + +const home = join(dirname(testAuthorityRegistryPath), 'home'); +process.env.HOME = home; +process.env.USERPROFILE = home; + +beforeAll(() => { + mkdirSync(home, { recursive: true }); +}); afterAll(releaseTestAuthorityRegistry); From 0b7762ff3b518bdf0ea604d6e5311c07728dc72f Mon Sep 17 00:00:00 2001 From: Nick Gomez <122398915+nick-inkeep@users.noreply.github.com> Date: Sat, 24 Oct 2026 00:57:41 +0000 Subject: [PATCH 2/2] Isolate clone stand-in tests from network overrides (#7498) * [wip] claim work on clone ambient environment fixture * Make clone stand-in network inputs hermetic GitOrigin-RevId: 2477b56e1c854e9968fc791cdf6c1c2f09bd5b58 --- knip.config.ts | 8 ++------ .../tests/integration/bug3-source-mode-writeback.test.ts | 1 - .../tests/integration/indented-jsx-byte-sacred.test.ts | 1 - .../app/tests/integration/interior-edit-freshness.test.ts | 1 - .../integration/keystroke-cadence-danger-space.test.ts | 1 - .../app/tests/integration/producer-guard-wiring.test.ts | 1 - .../qa-039-reconciliation-counters-http.test.ts | 1 - .../app/tests/integration/qa-045-restore-after-gc.test.ts | 1 - .../integration/qa-050-rollback-undo-shipped-path.test.ts | 1 - .../app/tests/integration/undo-after-rollback.test.ts | 1 - packages/app/tests/integration/undo-recycle-reset.test.ts | 1 - .../app/tests/integration/undo-rederive-survival.test.ts | 1 - packages/cli/src/commands/clone-git-config-env.test.ts | 3 +++ packages/core/src/bridge/compose.ts | 1 - packages/core/src/bridge/doc-boundary-space.ts | 1 - packages/core/src/markdown/comment-promoter.ts | 1 - packages/core/src/markdown/merged-walker.ts | 1 - packages/core/src/markdown/safe-url.ts | 1 - 18 files changed, 5 insertions(+), 22 deletions(-) diff --git a/knip.config.ts b/knip.config.ts index fac2580fe..e04bf5228 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -157,15 +157,11 @@ export default { 'src/shadow-fixture-launch.test-helper.mjs', ], project: 'src/**', - ignoreDependencies: [ - '@types/shell-quote', - ], + ignoreDependencies: ['@types/shell-quote'], }, 'packages/cli': { entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts', 'src/parse-worker.ts'], - ignoreDependencies: [ - 'yjs', - ], + ignoreDependencies: ['yjs'], }, 'packages/desktop': { entry: [ diff --git a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts index 4549c74b3..a50b00123 100644 --- a/packages/app/tests/integration/bug3-source-mode-writeback.test.ts +++ b/packages/app/tests/integration/bug3-source-mode-writeback.test.ts @@ -1,4 +1,3 @@ - import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; diff --git a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts index 4fc2c11f9..6fcdc23f2 100644 --- a/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts +++ b/packages/app/tests/integration/indented-jsx-byte-sacred.test.ts @@ -1,4 +1,3 @@ - import { setTimeout as wait } from 'node:timers/promises'; import { normalizeBridge } from '@inkeep/open-knowledge-core'; import { updateYFragment } from '@tiptap/y-tiptap'; diff --git a/packages/app/tests/integration/interior-edit-freshness.test.ts b/packages/app/tests/integration/interior-edit-freshness.test.ts index 9f2f6feb6..40b5cdfd6 100644 --- a/packages/app/tests/integration/interior-edit-freshness.test.ts +++ b/packages/app/tests/integration/interior-edit-freshness.test.ts @@ -1,4 +1,3 @@ - import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts index 2a085742b..93666af70 100644 --- a/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts +++ b/packages/app/tests/integration/keystroke-cadence-danger-space.test.ts @@ -1,4 +1,3 @@ - import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/producer-guard-wiring.test.ts b/packages/app/tests/integration/producer-guard-wiring.test.ts index 878d5196b..867598d34 100644 --- a/packages/app/tests/integration/producer-guard-wiring.test.ts +++ b/packages/app/tests/integration/producer-guard-wiring.test.ts @@ -1,4 +1,3 @@ - import { setTimeout as wait } from 'node:timers/promises'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; diff --git a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts index 4ada3cf26..0970ac12f 100644 --- a/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts +++ b/packages/app/tests/integration/qa-039-reconciliation-counters-http.test.ts @@ -1,4 +1,3 @@ - import { randomUUID } from 'node:crypto'; import { afterEach, describe, expect, test } from 'vitest'; import { agentWriteMd, createTestServer, type TestServer } from './test-harness'; diff --git a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts index cbe2b705f..b632185e1 100644 --- a/packages/app/tests/integration/qa-045-restore-after-gc.test.ts +++ b/packages/app/tests/integration/qa-045-restore-after-gc.test.ts @@ -1,4 +1,3 @@ - import { execFileSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { join } from 'node:path'; diff --git a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts index 89bc9b4d9..c530ce4ba 100644 --- a/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts +++ b/packages/app/tests/integration/qa-050-rollback-undo-shipped-path.test.ts @@ -1,4 +1,3 @@ - import { randomUUID } from 'node:crypto'; import { afterAll, afterEach, beforeAll, describe, expect, test } from 'vitest'; import type * as Y from 'yjs'; diff --git a/packages/app/tests/integration/undo-after-rollback.test.ts b/packages/app/tests/integration/undo-after-rollback.test.ts index a78aeb0e4..d3767adc9 100644 --- a/packages/app/tests/integration/undo-after-rollback.test.ts +++ b/packages/app/tests/integration/undo-after-rollback.test.ts @@ -1,4 +1,3 @@ - import { ROLLBACK_ORIGIN } from '@inkeep/open-knowledge-server'; import { afterAll, beforeAll, describe, expect, test } from 'vitest'; import * as Y from 'yjs'; diff --git a/packages/app/tests/integration/undo-recycle-reset.test.ts b/packages/app/tests/integration/undo-recycle-reset.test.ts index e7cc69c3f..b076f955c 100644 --- a/packages/app/tests/integration/undo-recycle-reset.test.ts +++ b/packages/app/tests/integration/undo-recycle-reset.test.ts @@ -1,4 +1,3 @@ - import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { setTimeout as wait } from 'node:timers/promises'; diff --git a/packages/app/tests/integration/undo-rederive-survival.test.ts b/packages/app/tests/integration/undo-rederive-survival.test.ts index 9ddcd0730..7935e35f8 100644 --- a/packages/app/tests/integration/undo-rederive-survival.test.ts +++ b/packages/app/tests/integration/undo-rederive-survival.test.ts @@ -1,4 +1,3 @@ - import { prependFrontmatter } from '@inkeep/open-knowledge-core'; import { Plugin, PluginKey } from '@tiptap/pm/state'; import { updateYFragment, yXmlFragmentToProseMirrorRootNode } from '@tiptap/y-tiptap'; diff --git a/packages/cli/src/commands/clone-git-config-env.test.ts b/packages/cli/src/commands/clone-git-config-env.test.ts index d58bf9963..75bc17e62 100644 --- a/packages/cli/src/commands/clone-git-config-env.test.ts +++ b/packages/cli/src/commands/clone-git-config-env.test.ts @@ -141,6 +141,9 @@ describe('clone honours the environment command-scope git config (GIT_CONFIG_COU vi.stubEnv('USERPROFILE', workspace); vi.stubEnv('XDG_CONFIG_HOME', join(workspace, '.config')); vi.stubEnv('GIT_CONFIG_NOSYSTEM', '1'); + vi.stubEnv('NO_PROXY', undefined); + vi.stubEnv('no_proxy', undefined); + vi.stubEnv('GIT_SSL_CAINFO', undefined); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); }); diff --git a/packages/core/src/bridge/compose.ts b/packages/core/src/bridge/compose.ts index 2a597ec3d..f4cc8a286 100644 --- a/packages/core/src/bridge/compose.ts +++ b/packages/core/src/bridge/compose.ts @@ -1,4 +1,3 @@ - import { prependFrontmatter, stripFrontmatter } from '../extensions/frontmatter.ts'; export type ComposeAdjustment = 'none' | 'body' | 'frontmatter' | 'unresolved'; diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts index ffd8fce06..e7e63f43d 100644 --- a/packages/core/src/bridge/doc-boundary-space.ts +++ b/packages/core/src/bridge/doc-boundary-space.ts @@ -1,4 +1,3 @@ - import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts'; import { carriedEdgeEmpties, type DocEdgeEmpties } from '../markdown/doc-edge-blank-runs.ts'; diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts index fb0ba5a7c..fc267cbea 100644 --- a/packages/core/src/markdown/comment-promoter.ts +++ b/packages/core/src/markdown/comment-promoter.ts @@ -1,4 +1,3 @@ - import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts index ae8cb390b..861003fb4 100644 --- a/packages/core/src/markdown/merged-walker.ts +++ b/packages/core/src/markdown/merged-walker.ts @@ -1,4 +1,3 @@ - import type { Nodes, Parent, Root } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts index 8be632ba4..4d3250d3a 100644 --- a/packages/core/src/markdown/safe-url.ts +++ b/packages/core/src/markdown/safe-url.ts @@ -1,4 +1,3 @@ - export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const; const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|');