diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 000000000..49a0213ef --- /dev/null +++ b/.mcp.json @@ -0,0 +1,5 @@ +{ + "mcpServers": { + "trail": { "command": "npx", "args": ["-y", "@trailhq/trail-code", "mcp"] } + } +} diff --git a/.trail/learnings/2026-10-29-the-mcp-registry-listing-reads-the-shrish.md b/.trail/learnings/2026-10-29-the-mcp-registry-listing-reads-the-shrish.md new file mode 100644 index 000000000..1f677623d --- /dev/null +++ b/.trail/learnings/2026-10-29-the-mcp-registry-listing-reads-the-shrish.md @@ -0,0 +1,34 @@ +--- +title: "The MCP Registry listing reads the npm package, so the repo keeps the graft package name" +author: Shrish +date: 2026-10-29 +branch: feat/mcp-directory-listing +cost: { minutes: 51, tokens: 345870 } +id: l-8e6ab5f6 +scope: repo +status: active +src: agent +touches: + - package.json + - scripts/pack-trail.mjs + - server.json + - scripts/pack-graft.mjs + - test/graft-to-trail.test.ts +--- + +## Decided +- Trail Code is listed in the official MCP Registry as io.github.trailhq/trail-code. The registry only accepts the entry when the npm package it names carries a matching mcpName, so only @trailhq/trail-code carries it. It is added where the trail package is laid out, not in the repo's package.json. +- Shrish decided (2026-10-30) the repo's package.json keeps the @nanonets/graft name. Echoic's listing checklist asked for the rename, but no directory needs it: the registry reads npm, Cursor Directory reads .mcp.json, and the rest are submitted by hand. + +## Tried and ruled out +- Renaming the repo's package to @trailhq/trail-code and laying out graft from it instead: 4 tests broke, because they assume this checkout runs as graft (the postinstall nudge, the upgrade advice under npx, the graft-to-trail move). +- One package with both the trail and graft commands, as the checklist suggested: anyone with both packages installed would get a clash on the graft command. + +## Watch out +- The registry entry must pass the `mcp` argument: bare `npx @trailhq/trail-code` starts the CLI, not the MCP server. Its description must be 100 characters or less. +- The entry names a fixed npm version and does not follow npm by itself. Bump both version fields and publish it again when a release should be what directories show. +- Publishing under io.github.trailhq/ needs `mcp-publisher login github` by a trailhq org member. + +In the code: +- scripts/pack-trail.mjs sets mcpName; server.json is the registry entry; .mcp.json at the root is for Cursor Directory. +- Check the entry with `mcp-publisher validate`, publish with `mcp-publisher publish`. diff --git a/.trail/learnings/2026-10-30-the-mcp-registry-listing-reads-the-shrish.md b/.trail/learnings/2026-10-30-the-mcp-registry-listing-reads-the-shrish.md new file mode 100644 index 000000000..85b20f9c0 --- /dev/null +++ b/.trail/learnings/2026-10-30-the-mcp-registry-listing-reads-the-shrish.md @@ -0,0 +1,37 @@ +--- +title: "The MCP Registry listing reads the npm package, so the repo keeps the graft package name" +author: Shrish +date: 2026-10-30 +branch: feat/mcp-directory-listing +cost: { minutes: 27, tokens: 274615 } +id: l-5a79b41c +scope: repo +status: active +replaces: l-8e6ab5f6 +src: agent +touches: + - package.json + - scripts/pack-trail.mjs + - server.json + - ".github/scripts/npm-downloads-badge.mjs" + - ".github/workflows/npm-downloads-badge.yml" + - README.md +--- + +## Decided +- Trail Code is listed in the official MCP Registry as io.github.trailhq/trail-code. The registry accepts the entry only when the npm package it names carries a matching mcpName, so only @trailhq/trail-code carries it, added where the trail package is laid out. +- Shrish decided (2026-10-30) the repo's package.json keeps the @nanonets/graft name. No directory needs the rename: the registry reads npm, Cursor Directory reads .mcp.json, the rest are submitted by hand. + +## Tried and ruled out +- Renaming the repo's package to @trailhq/trail-code: 4 tests broke, because they assume this checkout runs as graft. +- One package with both the trail and graft commands: installing both packages would clash on the graft command. +- Publishing after the plain GitHub device login: the registry refused with 403, because that login can't read the org role and only allows io.github./*. + +## Watch out +- Publishing under io.github.trailhq/ needs a trailhq owner signed in with a classic GitHub token that has read:org, or a GitHub Actions run (OIDC). The registry's sign-in lasts minutes, so run the login and the publish in one line. +- The entry must pass the `mcp` argument (bare `npx @trailhq/trail-code` starts the CLI), and its description must be 100 characters or less. +- The entry names a fixed npm version; bump both versions and publish it again when a release should be what directories show. + +In the code: +- scripts/pack-trail.mjs sets mcpName; server.json is the entry; .mcp.json is for Cursor Directory. +- `mcp-publisher validate`; `mcp-publisher login github --token && mcp-publisher publish`. diff --git a/.trail/learnings/2026-11-16-packing-the-graft-package-rebuilds-it-anirudhkumar.md b/.trail/learnings/2026-11-16-packing-the-graft-package-rebuilds-it-anirudhkumar.md new file mode 100644 index 000000000..a3fa12900 --- /dev/null +++ b/.trail/learnings/2026-11-16-packing-the-graft-package-rebuilds-it-anirudhkumar.md @@ -0,0 +1,27 @@ +--- +title: "Packing the graft package rebuilds it even with --ignore-scripts, which drops the telemetry key unless it's still set" +author: Anirudhkumar-nanonets +date: 2026-11-16 +branch: release/0.30.2 +id: l-87a404a2 +scope: repo +status: active +src: agent +touches: + - scripts/stamp-telemetry-key.mjs + - scripts/pack-trail.mjs + - package.json +--- + +## Decided +- Keep `GRAFT_POSTHOG_KEY` (and `GRAFT_POSTHOG_HOST`) set in the environment for the whole pack, not just the stamp, so the rebuild `npm pack` starts stamps the key again. Then check each tarball's key length before publishing, as the release order already says. +- Publishing a `.tgz` runs none of the package's scripts, so the tarball you checked is exactly what ships. +- 0.30.2 went out as `@trailhq/trail-code` only: the publishing account had no write access to `@nanonets/graft`, and the fix (share-session) only matters in the trail package. graft stayed at 0.30.1. +- Without the key at hand, it can be read from an installed earlier release's `dist/telemetry/key.js` (`posthogKey()`, `posthogHost()`) straight into the environment, never printed. + +## Watch out +- On npm 10.9.9, `npm pack --ignore-scripts` at the repo root still ran `prepare` (build, then stamp). With the key already unset, dist was rebuilt after the stamp and the graft tarball shipped an empty key, while `build/trail` (copied earlier by pack-trail) kept it. Nothing failed or warned; only the key-length check caught it. +- With 2FA set to auth-and-writes, `npm publish` waits for a browser approval and opens it only on Enter in that terminal. Give the person the printed npmjs.com/auth/cli link instead. +- The registry took about 5 minutes to list the new version after npm printed the `+ package@version` line. + +In the code: scripts/stamp-telemetry-key.mjs, scripts/pack-trail.mjs, package.json `prepare` (`npm run build && node scripts/stamp-telemetry-key.mjs`). diff --git a/.trail/learnings/2026-11-17-share-session-s-preview-is-four-anirudhkumar.md b/.trail/learnings/2026-11-17-share-session-s-preview-is-four-anirudhkumar.md new file mode 100644 index 000000000..2d8477ade --- /dev/null +++ b/.trail/learnings/2026-11-17-share-session-s-preview-is-four-anirudhkumar.md @@ -0,0 +1,32 @@ +--- +title: "share-session's preview is four short sections, then one line that says what to type: y, n, or folder numbers to leave out" +author: Anirudhkumar-nanonets +date: 2026-11-17 +branch: fix/share-ask-offers-leave-out +cost: { minutes: 42, tokens: 740869 } +id: l-9cb501d1 +scope: repo +status: active +replaces: l-69e4953a +src: agent +touches: + - src/cloud/share-bundle.ts + - src/cli.ts + - test/share-bundle.test.ts +--- + +## Decided +- The preview stays four headed sections: what's sent, from these folders, removed before sending, after you say yes. Its last line is where the copy is. A flat list of about 15 rows made a teammate miss the question entirely (16 Nov). +- After it comes one line saying what to type and what each answer does, then a bare `→` prompt: "Type y to send it all, n to send nothing, or folder numbers to leave out (like 2 3)." Enter still sends nothing. +- Folder numbers are offered whenever more than one folder is still in. With only "→ Send it? [y/N]", Aarsh (17 Nov) didn't know folders could be kept back, though typing numbers still worked. +- Leaving out can happen more than once: the preview is shown again with what's left, and the line asks about "the rest". The example numbers never name every folder still in, and a folder already left out can't be picked again. + +## Tried and ruled out +- "→ Send it? [y/N, or folder numbers to leave out]": two questions in one line. The user (17 Nov) asked for a line that says what to enter instead. +- Dropping the folder option from the question (16 Nov): people couldn't find it. +- "Not sent" as the heading of the removed list: too terse. The heading is "Removed before sending". + +## Watch out +- Review copy changes in a real terminal: run `node --import tsx src/bin/trail.ts share-session --once` under `script` with a throwaway HOME (a `~/.trail/cloud.json` pointing at a local stand-in that answers `/api/cli/shared-sessions/allowed` and the POST). Type the answers after the prompt shows: piped input ends first and reads as no. + +In the code: `sendAsk` and `parseLeaveOut` in src/cloud/share-bundle.ts; the answer loop in `shareWholeClaude` in src/cli.ts. diff --git a/.trail/learnings/2026-11-17-sharing-scrubs-a-session-line-too-anirudhkumar.md b/.trail/learnings/2026-11-17-sharing-scrubs-a-session-line-too-anirudhkumar.md new file mode 100644 index 000000000..6ddf1ab59 --- /dev/null +++ b/.trail/learnings/2026-11-17-sharing-scrubs-a-session-line-too-anirudhkumar.md @@ -0,0 +1,32 @@ +--- +title: "Sharing scrubs a session line too deep to walk as plain text, and leaves out a session it can't scrub instead of stopping" +author: Anirudhkumar-nanonets +date: 2026-11-17 +branch: fix/share-session-deep-nesting +cost: { minutes: 1, tokens: 1550 } +id: l-96699c8a +scope: repo +status: active +src: agent +touches: + - src/cloud/share-session.ts + - src/cloud/share-bundle.ts + - src/cloud/share-sync.ts + - src/trace/redact.ts +--- + +## Decided +- A session line nested more than 200 objects or arrays deep is scrubbed as plain text (`redactText`, then `anonymize`), the same way a line that isn't JSON is. Walking it with the recursive `walk`, and the `JSON.stringify` after it, ran out of stack at about 3,000 levels. That threw "Maximum call stack size exceeded" partway through scrubbing and ended the whole share with nothing sent. A teammate hit it on 17 Nov at session 29 of 83. +- A .env read nested that deep is left out (counted in `leftOut`): its values can only be found in parsed tool output, not in the raw text. +- A session the scrub still throws on never stops the run and is never sent unscrubbed. `prepareBundle` leaves it out and lists it in the manifest's `unreadable` and in SUMMARY.md. `runSync` skips it the way it skips a session that's too big, until the session grows. +- The scrub's regexes may only start where a run of their characters starts (a lookbehind): `IN_WORD`'s link-password pattern in redact.ts, and `EMAIL` in share-session.ts. Allowed to start at every letter, they re-read a long word to its end from each one, and a 1 to 2 MB word never finished. + +## Tried and ruled out +- `JSON.parse` as the limit: V8 parses a line nested 12,000 deep without trouble. The overflow comes from walking the parsed value and writing it back. +- Making only `walk` iterative: `JSON.stringify` of the cleaned value recurses too, so one depth check before the walk covers both. + +## Watch out +- A deep line scrubbed as text stays valid JSON unless `redactText` has to replace the whole line (`[redacted] line (…)`). Anything that reads shared bundles must accept non-JSON lines, as it already must for lines that weren't JSON to begin with. +- A new pattern in the scrub or in `secretReason` needs the same care: test it on a 1 MB run of the characters it matches. + +In the code: `nestsDeeper` and `MAX_WALK_DEPTH` in src/cloud/share-session.ts; `unreadable` and `scrub` in src/cloud/share-bundle.ts; `scrub` in src/cloud/share-sync.ts; `IN_WORD` in src/trace/redact.ts. Repro: one `.jsonl` line `{"type":"user","toolUseResult":{"a":{"a":…}}}` nested 3,000 deep, run through `bundleSession`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 67e26e9c2..fd0402c04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,67 @@ # Changelog -## Unreleased +## 0.30.5 + +- **`trail share-session` says what to type, and lets you leave folders + out.** After the preview, one line says what each answer does: `y` sends + it all, `n` sends nothing, and folder numbers leave those folders out + (like `2 3`). The preview then shows what's left, and you can leave out + more before sending (#847). + +## 0.30.4 + +- **`trail share-session` no longer stops on one unusual session.** A + session holding deeply nested JSON from a tool crashed the scrub with + "Maximum call stack size exceeded", and nothing was sent. A line that deep + is now scrubbed as text. A session the scrub still can't read is left out + and named in the summary, and the rest still go. Two scrub patterns no + longer slow to a halt on a very long word (#844). + +## 0.30.3 + +- **`trail share-session`'s preview is shorter and asks one question.** It + now shows four short sections: what's sent, from which folders, what's + removed before sending, and what happens after you say yes. Then it asks + `→ Send it? [y/N]`, where Enter means no. Typing folder numbers there + still leaves those folders out (#840). + +## 0.30.2 + +- **`trail share-session` keeps sending after a trail upgrade.** The first + Claude Code session after an upgrade rewrites trail's hooks in + `~/.claude/settings.json`, and that rewrite used to take out + share-session's hook with them. It now leaves it in place; only `--stop` + and `--delete` take it out (#838). + +## 0.30.1 + +- **`trail share-session` needs no flag.** It's still for the Trail team + only: Trail's server checks the signed-in email before anything is read. + On a laptop that isn't signed in, it opens the sign-in the way + `trail login` does, then carries on (#836). + +## 0.30.0 ### Added +- **Trail just for you.** `trail init` asks who trail is for. Enter keeps the + team setup that commits as before. "Just me" keeps trail's files out of git + for that clone and saves every learning in `~/.trail` (#750). +- **Claude Desktop and Codex.** AGENTS.md gets a Learnings part, and hooks + report the host they ran in, so learnings work in both (#733). +- **Stale learnings are flagged.** A learning whose `touches:` names code + that's gone is shown as "may be stale", with what's gone and how to replace + it. No model call (#731). +- **`trail init --claude-memory `** imports a Claude memory export. + It keeps only facts about code, never personal ones or secrets, and offers + them just for you (#732). +- **One learning file per session.** A later `trail note` in the same session + and branch adds to that file; a new branch starts a new one (#748). +- **`trail summary one-line`** turns each sources-diff block into one line + (#729). +- **New languages:** Julia, Bash, GLSL and PowerShell in the breadth tier, + `.svelte` and `.astro` through their script blocks, Elixir calls through + `alias`, and Pi as a host (#379, #354, #358, #550, #444, #351, #776). - **Structural graph failures can now fail the build command with `--strict`** instead of printing a successful wiring banner. Partial graph artifacts remain available, workspace builds report failed children, and `--allow-partial` @@ -21,6 +79,20 @@ ### Fixed +- **Your terminal title keeps the repo name.** Trail adds its status line + only when you don't already have one, and takes back the line an older + init wrote (#749, #756, #780). +- **Share sync re-sends a session only when it grew**, instead of on every + touched timestamp (#828). +- **More callers found.** TypeScript barrel re-exports, named imports, + interface-typed receivers and JSX components; Go package and interface + calls; Kotlin constructors; Python class receivers; PHP type usages + (#726, #335, #768, #764, #766, #687, #316, #339). +- **Windows:** no more console windows flashing from child processes, and the + post-edit blast radius shows (#683, #769, #755). +- The tokens-saved counter counts the learnings hooks hand over, so it keeps + moving in long sessions (#747). Telemetry events carry the time they were + sent, with the event's own time as `occurred_at` (#728). - Prevent telemetry helpers from repeatedly starting new copies when a sandbox blocks writes to the telemetry state. Internal flush commands skip startup upkeep, and background flushes require a saved timestamp before spawning. @@ -37,6 +109,46 @@ multi-clause heads collapse to one target. Measured on a 219-file app against `mix xref graph`: cross-file call-edge precision 0.38 → 0.93, recall 0.41 → 0.64. +## 0.29.0 + +### Added + +- **A sign-in link to your team's Trail.** At the end of Claude's reply in a + repo with learnings, trail now offers "sign in to Trail" by default. It + used to need `TRAIL_CLOUD=1`. It stays off with `DO_NOT_TRACK`, with + telemetry off, in CI, or with `TRAIL_CLOUD=0` (#730). +- **`trail share-session` can send your whole Claude setup**: the last 30 + days of Claude Code sessions, your memory notes and setup, scrubbed on your + laptop and sent only after you say yes. It's off unless + `TRAIL_SHARE_SESSION=1` is set. `--stop` and `--delete ` always work, + so anyone who tried it can take everything back (#740). + +### Fixed + +- **Background builds no longer pile up and hang a laptop.** The build lock + counted as abandoned after 5 minutes even while its build was still + running, so a slow laptop could run two or three builds at once. The lock + now names the process holding it, and background builds run at low CPU + priority (#736). +- **Lower peak memory on big repos.** Rebuilds after an edit run in their own + short-lived process instead of the MCP server, queries no longer copy the + whole search index, and map files are written as plain ASCII. On a repo + with 88k symbols, one turn peaked at 2.9 GB instead of 4.3 GB (#737). +- **Install and upgrade without sudo.** `trail upgrade` clears what a stopped + npm install leaves behind (ENOTEMPTY, EEXIST) and tries again, and on + EACCES it shows how to install into your home folder instead. On a Node + older than 20.16 or 22.3, `trail` stops with one plain line (#734). +- **Installing compiles nothing.** The depth tier (TypeScript, Python, Go, + Java, Kotlin, Swift, PHP, R) now parses on WASM, so a laptop without a C + toolchain installs cleanly. The map is the same (#707). +- `trail init`'s list of always-follow learnings says what each one makes + Claude do, and who gets it is said once as a heading (#724). + +## 0.28.1 + +- Lists Trail Code's MCP server in MCP directories (`mcpName`, `server.json`, + `.mcp.json`). The code is the same as 0.28.0. + ## 0.28.0 ### Added diff --git a/package-lock.json b/package-lock.json index 23f47c8d6..71d50d201 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@nanonets/graft", - "version": "0.28.0", + "version": "0.30.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@nanonets/graft", - "version": "0.28.0", + "version": "0.30.5", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 4c9b5a6b6..1f4b5caa9 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,13 @@ { "name": "@nanonets/graft", - "version": "0.28.0", + "version": "0.30.5", "description": "Build a repo's context graph as a folder of linked markdown files: a local, regenerable cache that every query keeps in sync with your code.", "keywords": [ + "mcp", + "mcp-server", + "model-context-protocol", + "claude-code", + "cursor", "ai", "agents", "knowledge-graph", diff --git a/scripts/pack-trail.mjs b/scripts/pack-trail.mjs index 913fdd046..721570fa7 100644 --- a/scripts/pack-trail.mjs +++ b/scripts/pack-trail.mjs @@ -35,6 +35,9 @@ const trail = { name: '@trailhq/trail-code', description: "Your team's shared memory for coding agents: a code map on your machine, and the learnings, decisions and skills your whole team shares. Formerly graft.", + // The official MCP Registry accepts server.json only when the npm package it + // names carries this, and it belongs to @trailhq/trail-code alone. + mcpName: 'io.github.trailhq/trail-code', keywords: [...new Set(['trail', ...(pkg.keywords ?? [])])], bin: { trail: 'dist/bin/trail.js' }, // Built already; nothing to compile in the published tree. diff --git a/server.json b/server.json new file mode 100644 index 000000000..f01b7426c --- /dev/null +++ b/server.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", + "name": "io.github.trailhq/trail-code", + "title": "Trail Code", + "description": "A local code map for coding agents: find code, trace callers, read a file's API, map a repo.", + "websiteUrl": "https://trailhq.com/trail-code", + "repository": { + "url": "https://github.com/trailhq/Trail-Code", + "source": "github", + "id": "1288325627" + }, + "version": "0.30.5", + "packages": [ + { + "registryType": "npm", + "identifier": "@trailhq/trail-code", + "version": "0.30.5", + "transport": { + "type": "stdio" + }, + "packageArguments": [ + { + "type": "positional", + "value": "mcp" + } + ] + } + ] +} diff --git a/src/cli.ts b/src/cli.ts index 8733e339a..16e39707d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -130,7 +130,7 @@ import { trackFirstRunIfNew, } from "./telemetry/index.js"; import { bundleSession, listLines, MAX_GZ_BYTES, pickSession, previewLines, redactedCount, sessionsHere, shortId, sizeLabel, whoIAm } from "./cloud/share-session.js"; -import { bundlePreviewLines, deleteCode, deleteHash, itemPayload, leaveOut, newCopyDir, newSentState, parseDeleteCode, parseLeaveOut, prepareBundle, readSent, SEND_ASK, SEND_REST_ASK, unfinishedCopy, writeSent, type Manifest, type SentState } from "./cloud/share-bundle.js"; +import { bundlePreviewLines, deleteCode, deleteHash, itemPayload, leaveOut, newCopyDir, newSentState, parseDeleteCode, parseLeaveOut, prepareBundle, readSent, sendAsk, unfinishedCopy, writeSent, type Manifest, type SentState } from "./cloud/share-bundle.js"; import { trailHome } from "./notes/trail-home.js"; import { readSyncState, runSync, syncDir, writeSyncState, type SyncState } from "./cloud/share-sync.js"; import { hasShareSyncHook, installShareSyncHook, removeShareSyncHook, settingsPath, type HookResult } from "./cloud/share-hook.js"; @@ -2818,8 +2818,9 @@ async function shareWholeClaude(cloud: Cloud, opts: { days: number; leaveOut?: s track("share_bundle", { outcome: "declined", items_bucket: shareItemsBucket(m.items.length), size_bucket: shareSizeBucket(bundleGz(m)) }, { repo: checkout }); console.error("· nothing sent, and the copy is deleted"); }; + // Asked until y or n: folders can be left out in more than one go, and the preview shows what's left each time. for (;;) { - const a = await askLine(SEND_ASK); + const a = await askLine(sendAsk(m, m.dropped.length > 0)); if (a === null || /^(n|no)?$/i.test(a)) return declined(); if (/^y(es)?$/i.test(a)) return void (await sendCopy(cloud, out, m, newSentState(), checkout, opts.sync)); const ns = parseLeaveOut(a, m); @@ -2829,8 +2830,6 @@ async function shareWholeClaude(cloud: Cloud, opts: { days: number; leaveOut?: s } m = leaveOut(out, m, ns); show(); - if (await askYes(SEND_REST_ASK)) return void (await sendCopy(cloud, out, m, newSentState(), checkout, opts.sync)); - return declined(); } } diff --git a/src/cloud/share-bundle.ts b/src/cloud/share-bundle.ts index 8a06aaba3..a46775a58 100644 --- a/src/cloud/share-bundle.ts +++ b/src/cloud/share-bundle.ts @@ -53,6 +53,8 @@ export interface Manifest { dropped: string[]; /** Sessions over the upload cap, left out. */ tooBig: Array<{ folder: string; title: string; gzBytes: number }>; + /** Sessions the scrub failed on, left out. Absent in a copy made before trail kept it. */ + unreadable?: Array<{ folder: string; id: string; mtime: number }>; /** Secrets taken out, by kind. */ redacted: Record; /** Places the person's name, email or home folder was taken out. */ @@ -105,12 +107,16 @@ export function prepareBundle(opts: { now?: number; repoOf?: (cwd: string) => string; onProgress?: (done: number, total: number) => void; + /** The scrub, bundleSession unless a test swaps it. */ + scrub?: typeof bundleSession; }): Manifest { const now = opts.now ?? Date.now(); + const scrub = opts.scrub ?? bundleSession; const sessions = sessionsSince(opts.days, opts.home, now); mkdirSync(opts.out, { recursive: true }); const items: BundleItem[] = []; const tooBig: Manifest["tooBig"] = []; + const unreadable: NonNullable = []; const redacted: Record = {}; const add = (r: Record) => { for (const [k, n] of Object.entries(r)) redacted[k] = (redacted[k] ?? 0) + n; @@ -120,9 +126,18 @@ export function prepareBundle(opts: { let leftOut = 0; sessions.forEach((s, i) => { - const b = bundleSession(s, opts.who); const cwd = sessionCwd(s); const folder = cwd ? anonymize(cwd, opts.who).text : (s.folder ?? ""); + let b: ReturnType; + try { + b = scrub(s, opts.who); + } catch { + // One session the scrub can't read is left out and named in the summary, never sent + // unscrubbed; the rest still go. Before, it ended the whole share with nothing sent. + unreadable.push({ folder, id: s.id, mtime: s.mtime }); + opts.onProgress?.(i + 1, sessions.length); + return; + } hidden += b.hidden; leftOut += b.leftOut; if (b.gz.length > MAX_GZ_BYTES) { @@ -182,6 +197,7 @@ export function prepareBundle(opts: { folders: numberFolders(items), dropped: [], tooBig, + unreadable, redacted, hidden, leftOut, @@ -208,7 +224,8 @@ export function parseLeaveOut(answer: string, m: Manifest): number[] | null { const parts = answer.split(/[\s,]+/).filter(Boolean); if (!parts.length || !parts.every((p) => /^\d{1,4}$/.test(p))) return null; const ns = [...new Set(parts.map(Number))]; - return ns.every((n) => m.folders.some((f) => f.n === n)) ? ns : null; + // Only folders still in: one already left out is no longer on the list. + return ns.every((n) => m.folders.some((f) => f.n === n && !m.dropped.includes(f.label))) ? ns : null; } /** Take the numbered folders out of the copy: their files go, and the summary says so. */ @@ -295,16 +312,27 @@ function plural(n: number, one: string, many = `${one}s`): string { return `${n} ${n === 1 ? one : many}`; } -/** The question after the preview: always its last line, with Enter meaning no. */ -export const SEND_ASK = "→ Send it? [y/N] "; -/** Asked again after folders were left out. */ -export const SEND_REST_ASK = "→ Send the rest? [y/N] "; +/** + * What to type after the preview: one line that says each answer and what it + * does, then the `→` prompt, so nobody has to guess what a question with three + * answers wants. Folder numbers are offered while more than one folder is in: + * with "→ Send it? [y/N]" alone, a teammate didn't know folders could be kept + * back. Enter still sends nothing. `rest`: asked again after some were left out. + */ +export function sendAsk(m: Manifest, rest = false): string { + const kept = m.folders.filter((f) => !m.dropped.includes(f.label)); + const what = rest ? "the rest" : kept.length > 1 ? "it all" : "it"; + if (kept.length < 2) return `Type y to send ${what}, or n to send nothing.\n→ `; + // The example never names every folder still in: that would send nothing but the setup. + const example = kept.slice(kept.length > 2 ? -2 : -1).map((f) => f.n).join(" "); + return `Type y to send ${what}, n to send nothing, or ${rest ? "more " : ""}folder numbers to leave out (like ${example}).\n→ `; +} /** * What will be sent, for the person to say yes to, in four short sections: * what's sent, from which folders, what isn't, and what happens after yes. * `copy` is the folder as shown (`~/.trail/share/…`). The send question - * (`SEND_ASK`) follows it, so nothing here may read like the last line. + * (`sendAsk`) follows it, so nothing here may read like the last line. */ export function bundlePreviewLines(m: Manifest, copy: string, to: { url: string }, opts: { sync?: string } = {}): string[] { const sessions = m.items.filter((it) => it.kind === "session"); @@ -334,6 +362,7 @@ export function bundlePreviewLines(m: Manifest, copy: string, to: { url: string ...(m.setup.mcp ? ["your MCP servers' env values and headers"] : []), ...(m.leftOut ? [plural(m.leftOut, "image or other file that isn't text", "images and other files that aren't text")] : []), ...(m.tooBig.length ? [`${plural(m.tooBig.length, "session")} over ${sizeLabel(MAX_GZ_BYTES)} zipped, too big to send`] : []), + ...(m.unreadable?.length ? [`${plural(m.unreadable.length, "session")} trail couldn't scrub, so it isn't sent`] : []), ...(m.dropped.length ? [`${plural(m.dropped.length, "folder")} you left out: ${m.dropped.join(", ")}`] : []), ]; return [ @@ -413,5 +442,10 @@ function writeCopy(dir: string, m: Manifest): void { for (const t of m.tooBig) out.push(`- ${t.folder} · ${t.title || "(nothing typed)"} · ${sizeLabel(t.gzBytes)} zipped`); out.push(""); } + if (m.unreadable?.length) { + out.push("## Left out: trail couldn't scrub these", ""); + for (const u of m.unreadable) out.push(`- ${u.folder} · ${dayLabel(u.mtime)} · session ${u.id.slice(0, 8)}`); + out.push(""); + } writeFileSync(join(dir, "SUMMARY.md"), `${out.join("\n")}\n`); } diff --git a/src/cloud/share-session.ts b/src/cloud/share-session.ts index e83d682a7..1ccbb5ac7 100644 --- a/src/cloud/share-session.ts +++ b/src/cloud/share-session.ts @@ -260,6 +260,19 @@ export function bundleSession(session: SessionFile, who: string[] = whoIAm()): B if (endedAt === undefined || at > Date.parse(endedAt)) endedAt = e.timestamp as string; } const env = touchesEnv(entry, envIds); + // A tool's output can be JSON nested thousands deep. Walking it, or writing it back, runs + // out of stack and stopped the whole share, so a line that deep is scrubbed as text, like + // a line that isn't JSON. A .env read that deep is left out: its values can't be found in text. + if (nestsDeeper(line, MAX_WALK_DEPTH)) { + if (env) { + leftOut++; + continue; + } + const r = redactText(line); + count(r.found); + out.push(hide(r.text)); + continue; + } const clean = walk(entry, (s) => { let text = s; if (env) { @@ -336,7 +349,9 @@ export function whoIAm(extra: string[] = []): string[] { /** Machine logins that name nobody, and would turn up as ordinary words (`test runner`). */ const GENERIC_LOGINS = new Set(["root", "admin", "user", "users", "ubuntu", "runner", "vscode", "node", "ec2-user", "codespace", "developer", "dev", "app", "default"]); -const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g; +// Starts only where a run of address characters starts: tried at every position of a long +// run with no `@`, it read to the run's end each time, and a 1 MB word never finished. +const EMAIL = /(?): boolean { /** A long base64 run: an image or a file's bytes, not words. */ const BASE64 = /^[A-Za-z0-9+/=\r\n]{512,}$/; +/** + * How deep `walk` goes into one line. Claude Code's own lines nest about 10 + * deep; walking past a few thousand, and `JSON.stringify` after it, overflows + * the stack. Well under that, for a hook's or a Windows thread's smaller stack. + */ +const MAX_WALK_DEPTH = 200; + +/** + * Whether the JSON `line` nests deeper than `max` objects and arrays. One pass + * over the text, without recursing, so a line too deep to walk is told apart + * without walking it. + */ +export function nestsDeeper(line: string, max: number): boolean { + if (line.length <= max) return false; + let depth = 0; + let inString = false; + for (let i = 0; i < line.length; i++) { + const c = line.charCodeAt(i); + if (inString) { + if (c === 0x5c) i++; // a backslash escapes the next character + else if (c === 0x22) inString = false; + } else if (c === 0x22) inString = true; + else if (c === 0x7b || c === 0x5b) { + if (++depth > max) return true; + } else if (c === 0x7d || c === 0x5d) depth--; + } + return false; +} + /** * `value` with `fix` applied to every string in it. An image's bytes are left * out (`dropped` counts them), and so are a thinking block's signature and a diff --git a/src/cloud/share-sync.ts b/src/cloud/share-sync.ts index 7f7fb17ef..5d02afc54 100644 --- a/src/cloud/share-sync.ts +++ b/src/cloud/share-sync.ts @@ -86,6 +86,8 @@ export async function runSync(opts: { repoOf?: (cwd: string) => string; env?: NodeJS.ProcessEnv; now?: number; + /** The scrub, bundleSession unless a test swaps it. */ + scrub?: typeof bundleSession; }): Promise<{ sent: number; skipped: number; error?: string } | null> { const now = opts.now ?? Date.now(); const s = readSyncState(opts.env); @@ -105,8 +107,15 @@ export async function runSync(opts: { const cwd = sessionCwd(f); const folder = cwd ? anonymize(cwd, opts.who).text : (f.folder ?? ""); if (cwd && existsSync(cwd)) cwds.add(cwd); - const b = bundleSession(f, opts.who); - if (s.excluded.includes(folder) || b.gz.length > MAX_GZ_BYTES) { + // A session the scrub fails on is skipped like one too big, until it grows: it never goes + // unscrubbed, and it doesn't stop the sessions after it. + let b: ReturnType | null = null; + try { + b = (opts.scrub ?? bundleSession)(f, opts.who); + } catch { + b = null; + } + if (!b || s.excluded.includes(folder) || b.gz.length > MAX_GZ_BYTES) { sentAs(s, f); skipped++; writeSyncState(s, opts.env); diff --git a/src/trace/redact.ts b/src/trace/redact.ts index 9ed67a3da..c6f4baec5 100644 --- a/src/trace/redact.ts +++ b/src/trace/redact.ts @@ -57,9 +57,13 @@ export const REDACTED = '[redacted]'; const PRIVATE_KEY_BLOCK = /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----|$)/g; /** A word, for finding where on a line the secret is: a run between spaces, quotes and brackets. `=`, `:` and `/` stay in it. */ const WORD = /[^\s'"`,;{}()[\]<>]+/g; -/** The ways to take a secret out of one word and keep the rest, tried in order: a link's password, a link's key, a name's value. */ +/** + * The ways to take a secret out of one word and keep the rest, tried in order: a link's password, a link's key, a name's value. + * The link's scheme starts only where a run of scheme characters starts: tried at every letter of a long word, it read to + * the word's end each time, and a 2 MB word never finished. + */ const IN_WORD: Array<[RegExp, string]> = [ - [/([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s@/]+(?=@)/gi, `$1${REDACTED}`], + [/(? { + const home = fakeHome(); + const out = join(home, ".trail", "share", "2026-11-05-1200"); + const bad = "aaaa1111-0000-4000-8000-000000000002"; + const m = prepareBundle({ + days: 30, + home, + who: [home], + out, + now: NOW, + scrub: (s, who) => { + if (s.id === bad) throw new RangeError("Maximum call stack size exceeded"); + return bundleSession(s, who); + }, + }); + assert.deepEqual( + m.items.filter((it) => it.kind === "session").map((it) => it.id).sort(), + ["aaaa1111-0000-4000-8000-000000000001", "bbbb2222-0000-4000-8000-000000000001"], + "one session stopped nothing: before, the whole share ended there with nothing sent", + ); + assert.deepEqual(m.unreadable, [{ folder: "/work/web", id: bad, mtime: NOW - 2 * 24 * 60 * 60 * 1000 }]); + assert.ok(!existsSync(join(out, "sessions", "work-web", `${bad}.jsonl.gz`)), "nothing of it is written, so nothing of it can go"); + assert.match(readFileSync(join(out, "SUMMARY.md"), "utf8"), /^## Left out: trail couldn't scrub these\n\n- \/work\/web · Nov 3 · session aaaa1111$/m); + assert.match(bundlePreviewLines(m, "~/x", { url: "https://team.trailhq.com" }).join("\n"), /^ {2}1 session trail couldn't scrub, so it isn't sent$/m); +}); + test("bundlePreviewLines: what's sent, from which folders, what's removed, what happens after yes, then where the copy is", () => { const { m } = prepared(); const lines = bundlePreviewLines(m, "~/.trail/share/2026-11-05-1200", { url: "https://team.trailhq.com" }); @@ -112,6 +140,19 @@ test("parseLeaveOut and leaveOut: folder numbers drop those folders' files from assert.deepEqual(readdirSync(join(out, "sessions")), ["work-web"]); assert.match(readFileSync(join(out, "SUMMARY.md"), "utf8"), /2\. \/work\/api · 1 session · left out/); assert.match(bundlePreviewLines(next, "~/x", { url: "https://team.trailhq.com" }).join("\n"), /^ {2}1 folder you left out: \/work\/api$/m); + assert.equal(parseLeaveOut("2", next), null, "a folder already left out is no longer on the list"); +}); + +test("sendAsk: says what to type, folder numbers included while more than one folder is in, then asks about the rest", () => { + const { out, m } = prepared(); + // A teammate saw only "→ Send it? [y/N]" and asked how to keep a folder back: the numbers worked, nothing said so. + assert.equal(sendAsk(m), "Type y to send it all, n to send nothing, or folder numbers to leave out (like 2).\n→ "); + const one = leaveOut(out, m, [2]); + assert.equal(sendAsk(one, true), "Type y to send the rest, or n to send nothing.\n→ ", "one folder left: leaving it out would send nothing"); + assert.equal(sendAsk({ ...m, folders: [m.folders[0]!] }), "Type y to send it, or n to send nothing.\n→ "); + const three = { ...m, folders: [...m.folders, { n: 3, label: "/work/docs", sessions: 1 }] }; + assert.equal(sendAsk(three), "Type y to send it all, n to send nothing, or folder numbers to leave out (like 2 3).\n→ ", "the example never names every folder"); + assert.equal(sendAsk({ ...three, dropped: ["/work/api"] }, true), "Type y to send the rest, n to send nothing, or more folder numbers to leave out (like 3).\n→ "); }); test("itemPayload: a session goes as its zipped file, the setup zipped on the way", () => { diff --git a/test/share-session.test.ts b/test/share-session.test.ts index 114daabfa..f7b1c8c72 100644 --- a/test/share-session.test.ts +++ b/test/share-session.test.ts @@ -16,7 +16,8 @@ import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { gunzipSync } from "node:zlib"; import { BRAND_ENV } from "../src/brand.js"; -import { anonymize, bundleSession, pickSession, previewLines, redactedCount, sessionsHere, whoIAm, type SessionFile } from "../src/cloud/share-session.js"; +import { anonymize, bundleSession, nestsDeeper, pickSession, previewLines, redactedCount, sessionsHere, whoIAm, type SessionFile } from "../src/cloud/share-session.js"; +import { timeBar } from "./machine-speed.js"; import { deleteSharedBundle, shareAllowed, shareSession, type Cloud } from "../src/cloud/workspace.js"; import { claudeProjectDir } from "../src/notes/session-cost.js"; import { secretReason } from "../src/notes/secrets.js"; @@ -163,6 +164,61 @@ test("a .env file counts as read only when a command reads it, not when its name assert.equal(result(4), "PORT=[redacted]\nREGION=[redacted]"); }); +test("bundleSession: a line nested thousands deep goes scrubbed as text, and the rest of the session still goes", () => { + const home = mkdtempSync(join(tmpdir(), "share-deep-")); + const folder = join(home, ".claude", "projects", "-work-web"); + mkdirSync(folder, { recursive: true }); + const id = "dddd1111-0000-4000-8000-000000000001"; + // A tool's output can be any JSON; walking 5,000 levels ran out of stack and stopped the whole share. + let nested = JSON.stringify(`rotate ${OPENAI} in /Users/maya/web`); + for (let i = 0; i < 5000; i++) nested = `{"a":${nested}}`; + const lines = [ + JSON.stringify({ type: "user", timestamp: "2026-11-17T06:00:00.000Z", sessionId: id, cwd: "/work/web", message: { role: "user", content: "parse the tree" } }), + `{"type":"user","timestamp":"2026-11-17T06:01:00.000Z","toolUseResult":${nested},"message":{"role":"user","content":"tool output"}}`, + JSON.stringify({ type: "assistant", timestamp: "2026-11-17T06:02:00.000Z", message: { id: "msg_02", role: "assistant", content: [{ type: "text", text: "done" }] } }), + ]; + const path = join(folder, `${id}.jsonl`); + writeFileSync(path, lines.join("\n") + "\n"); + + const b = bundleSession({ id, path, mtime: Date.now(), size: 0 }, ["/Users/maya", "maya"]); + const text = gunzipSync(b.gz).toString("utf8"); + const out = text.trim().split("\n"); + assert.equal(out.length, 3, "every line goes"); + assert.equal(b.messages, 3, "the deep line still counts as a message"); + assert.equal(secretReason(text), null, "the key in the deep line is gone"); + assert.doesNotMatch(text, /maya/, "so is the sender's home folder"); + assert.equal(JSON.parse(out[0]!).message.content, "parse the tree", "the lines around it are walked as before"); + assert.equal(JSON.parse(out[2]!).message.content[0].text, "done"); +}); + +test("nestsDeeper: counts objects and arrays, not brackets inside strings", () => { + assert.equal(nestsDeeper('{"a":[{"b":1}]}', 3), false); + assert.equal(nestsDeeper('{"a":[{"b":[1]}]}', 3), true); + assert.equal(nestsDeeper(JSON.stringify({ s: "[[[[[[[[{{{{{{{{" }), 2), false, "brackets in a string don't nest"); + assert.equal(nestsDeeper(JSON.stringify({ s: 'a \\" [[[[ "' }), 2), false, "nor after an escaped quote"); +}); + +test("redactText and anonymize read a 1 MB word once, not once per letter", () => { + const word = "a".repeat(1024 * 1024); + // Each of these never finished before: a pattern that could start at any letter read to the word's end from each. + const runs: Array<[string, () => unknown]> = [ + ["a link's key then a long value", () => redactText(`https://x.test/?token=${word}`)], + ["a secret's name then a long value", () => redactText(`TOKEN=${word}`)], + ["a long word with no @", () => anonymize(word, ["maya@acme.test", "maya"])], + ]; + const bar = timeBar(250, 1500); + for (const [name, run] of runs) { + const t0 = performance.now(); + run(); + const ms = performance.now() - t0; + assert.ok(ms < bar, `${name}: ${ms.toFixed(0)} ms, over ${bar.toFixed(0)} ms`); + } + assert.equal(redactText(`https://x.test/?token=${word}`).text, `https://x.test/?token=${REDACTED}`); + assert.equal(redactText(`TOKEN=${word}`).text, `TOKEN=${REDACTED}`); + assert.equal(anonymize("mail maya.k+ci@acme.test now", []).text, "mail [email] now", "an address is still found whole"); + assert.equal(redactText("postgres://app:Xk9vQ2mL7pR4@db.acme.test/main").text, `postgres://app:${REDACTED}@db.acme.test/main`, "and a link's password"); +}); + test("pickSession: by number in the list, by id or its start; an unknown or ambiguous one is null", () => { const s = (id: string): SessionFile => ({ id, path: `/x/${id}.jsonl`, mtime: 0, size: 1 }); const list = [s("aaaa1111"), s("aaaa2222"), s("bbbb3333")]; diff --git a/test/share-sync.test.ts b/test/share-sync.test.ts index acb9e1196..952e65f8b 100644 --- a/test/share-sync.test.ts +++ b/test/share-sync.test.ts @@ -15,6 +15,7 @@ import { newSentState } from "../src/cloud/share-bundle.js"; import { changedSessions, maybeShareSyncInBackground, readSyncState, runSync, syncDir, syncDue, writeSyncState, type SyncState } from "../src/cloud/share-sync.js"; import { QUIET_MS } from "../src/cloud/share-sync-state.js"; import type { SharedSession } from "../src/cloud/workspace.js"; +import { bundleSession } from "../src/cloud/share-session.js"; import { secretReason } from "../src/notes/secrets.js"; const OPENAI = "sk-" + "proj-" + "Q7x2Lm9Pz4Rt8Vb1Nc6Kw3Hy5Jd0Fs"; @@ -152,6 +153,26 @@ test("runSync: a send that fails stops there and keeps what went; the next run s assert.deepEqual(next.got.map((b) => b.kind), ["session", "setup"]); }); +test("runSync: a session the scrub fails on is skipped until it grows, and the rest still go", async () => { + const { home, env } = synced(); + const c = collector(); + const r = await runSync({ + home, + who: [home], + send: c.send, + trailVersion: "0.29.0", + env, + now: NOW, + scrub: (f, who) => { + if (f.id === "aaaa0000-0000-4000-8000-000000000002") throw new RangeError("Maximum call stack size exceeded"); + return bundleSession(f, who); + }, + }); + assert.deepEqual(r, { sent: 1, skipped: 2 }, "the failing session and the left-out folder are skipped; the setup still goes"); + assert.deepEqual(c.got.map((b) => b.kind), ["setup"]); + assert.ok(readSyncState(env)!.sessions["aaaa0000-0000-4000-8000-000000000002"], "it isn't tried again on every run"); +}); + test("runSync: one run at a time, and none once stopped or never turned on", async () => { const { home, env, state } = synced(); mkdirSync(syncDir(env), { recursive: true });