From e30bda124baae2920774756667254622d125a4bb Mon Sep 17 00:00:00 2001 From: Alexey Grigorev Date: Thu, 5 Nov 2026 00:51:45 +0100 Subject: [PATCH] 9ff3f391: reconcile upload accepts Finom CSV statements The reconcile flow could parse Finom CSV exports only through the email intake; the CLI, portal API and document upload path were PDF-only at every layer, so a Finom statement CSV had no way into reconciliation. - recognizeStatement now recognizes tabular exports from their header row against the same fixed layouts the email intake already uses (Finom statement, Revolut), including the selected-account mismatch check; the email intake keeps its behavior, with errors that now carry the filename. - prepareStatementImport commits an identity-less tabular statement only when the operator explicitly selected the account (--account), which stands in for the missing IBAN; without it the source stays at review, as before. Only a source with header identity refuses to coexist with an unverified overlapping source, so re-uploads of the same file replay to the same statement id. - Bank-statement documents accept text/csv at prepare and completion (the object must be readable text, not a renamed PDF or binary); XLSX and every other document type stay PDF-only. - dataops reconcile upload (alias: upload-statement) accepts statement.pdf or statement.csv, validating bytes locally before any upload; help documents the --account rule for CSV exports. Scope: in --- backend/src/reconcile/statementImport.ts | 13 +- backend/src/reconcile/statementIntake.ts | 68 ++++---- backend/src/routes/bookkeepingDocuments.ts | 12 +- backend/src/routes/bookkeepingShared.ts | 18 ++- .../bookkeeping-document-atomic-api.test.ts | 63 +++++--- backend/tests/reconcile.test.ts | 153 +++++++++++++----- cli/src/commands/reconcile.mjs | 16 +- cli/src/documentUpload.mjs | 18 ++- cli/test/reconcile.test.mjs | 81 ++++++++-- 9 files changed, 303 insertions(+), 139 deletions(-) diff --git a/backend/src/reconcile/statementImport.ts b/backend/src/reconcile/statementImport.ts index 062171811..a8e562507 100644 --- a/backend/src/reconcile/statementImport.ts +++ b/backend/src/reconcile/statementImport.ts @@ -12,7 +12,7 @@ import { coverageGaps } from "./match"; export class StatementIdentityReviewError extends Error { statusCode = 422; - constructor(readonly account: StatementAccount) { super("Statement header has no stable bank account identifier; an IBAN or account number is required before import. Source retained for review; no active statement changed."); } + constructor(readonly account: StatementAccount, message?: string) { super(message || "Statement header has no stable bank account identifier; an IBAN or account number is required before import. Source retained for review; no active statement changed."); } } export type StatementImportOptions = { preview?: boolean; replace?: boolean; previewDigest?: string; expectedAccount?: StatementAccount; expectedPeriod?: string; by?: string }; @@ -70,7 +70,10 @@ export async function prepareStatementImport(client: DynamoDBDocumentClient, sou if (options.expectedPeriod && !periods.includes(options.expectedPeriod)) throw new Error("Statement header/rows do not cover the selected month"); if (periods.length > 24) throw new Error("Import at most 24 months in one statement batch"); const missingIdentity = [...parsed.rows, ...parsed.balances].some((row) => !parsed.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]); - if (missingIdentity) throw new StatementIdentityReviewError(account); + // Tabular exports (Finom CSV) print no IBAN. They commit only when the + // operator explicitly selected the account, which stands in for the header + // identity; otherwise the source waits for review, as the email intake does. + if (missingIdentity && !options.expectedAccount) throw new StatementIdentityReviewError(account); const history = await listStatementHistory(client); const months = await Promise.all(periods.map((period) => getMonth(client, period))); const skippedClosedMonths = months.filter((month) => month.state === "closed").map((month) => month.period); @@ -106,7 +109,11 @@ export async function prepareStatementImport(client: DynamoDBDocumentClient, sou const candidate: StatementRecord = { id, recordType: "reconcile-statement", accountId: account, period, ...source, rows, balances, ...bounds, bankAccounts: parsed.bankAccounts, ...(parsed.issueDate ? { issueDate: parsed.issueDate } : {}), ...(parsed.openingBalances ? { openingBalances: parsed.openingBalances.filter((balance) => balance.asOf.startsWith(period)) } : {}), warnings: parsed.warnings, createdAt: previous?.createdAt || now, updatedAt: now }; - if (verified.some((record) => record.accountId === account && overlaps(record, candidate) && [...record.rows, ...(record.balances || [])].some((row) => !record.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]))) throw new StatementIdentityReviewError(account); + // Only a source with header identity may not coexist with an unverified one; two + // operator-asserted (identity-less) sources of the same account overlap harmlessly. + const candidateVerified = [...rows, ...balances].every((row) => parsed.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]); + if (candidateVerified && verified.some((record) => record.accountId === account && overlaps(record, candidate) && [...record.rows, ...(record.balances || [])].some((row) => !record.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]))) + throw new StatementIdentityReviewError(account, "An overlapping statement source of this account carries no bank account identifier; delete or replace that source before importing a statement with verified identity. No active statement changed."); const keys = [...new Set([...rows, ...balances].map((row) => walletKey(candidate, row)))].sort(); const existing = active.flatMap((record) => { if (record.accountId !== account || !overlaps(record, candidate)) return []; diff --git a/backend/src/reconcile/statementIntake.ts b/backend/src/reconcile/statementIntake.ts index c7967d789..aef8f4442 100644 --- a/backend/src/reconcile/statementIntake.ts +++ b/backend/src/reconcile/statementIntake.ts @@ -114,42 +114,35 @@ function pdfFailure(error: unknown, text: string, filename?: string): ParseError return new ParseError(`${safeFilename(filename)}: line ${source.number}, source text "${excerpt}": ${message}`); } -/** Recognize and completely parse a supported bank PDF before any statement rows are stored. */ -export async function recognizeStatement(bytes: Buffer, contentType: string, options: { currency?: string; expectedAccount?: StatementAccount; filename?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> { - const type = contentType.split(";")[0].trim().toLowerCase(); - if (!isPdf(type, bytes)) { - throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: bank statement uploads accept PDF files only`); - } - - let text: string; - try { text = await statementPdfText(bytes); } - catch (error) { throw pdfFailure(error, "", options.filename); } - const account = detectPdfStatementAccount(text); - if (!account) { - const message = "PDF is not a recognized Finom or Revolut statement"; - throw new UnrecognizedStatementError(pdfFailure(message, text, options.filename).message); - } - if (options.expectedAccount && account !== options.expectedAccount) { - throw pdfFailure(`PDF contains a ${account} statement, not the selected ${options.expectedAccount} account`, text, options.filename); - } - - try { - const parsed = account === "finom" ? parseFinomPdfText(text) : parseRevolutPdfText(text); - if (!parsed.rows.length) throw new ParseError("PDF statement parsed zero transaction rows"); - return { account, layout: `${account} PDF statement`, parsed }; - } catch (error) { - throw pdfFailure(error, text, options.filename); - } -} - /** - * Email document intake keeps its existing tabular attachment handling. - * The explicit reconcile upload API, CLI and portal use recognizeStatement, - * which accepts PDFs only. + * Recognize and completely parse a supported bank statement before any rows + * are stored. PDFs are read through their bank's fixed layout; tabular + * exports (CSV, and XLSX from the email intake) are recognized from their + * header row against the same fixed layouts. Used by the explicit reconcile + * upload API, CLI, portal and the email intake. */ -async function recognizeEmailStatement(bytes: Buffer, contentType: string, options: { currency?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> { +export async function recognizeStatement(bytes: Buffer, contentType: string, options: { currency?: string; expectedAccount?: StatementAccount; filename?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> { const type = contentType.split(";")[0].trim().toLowerCase(); - if (isPdf(type, bytes)) return recognizeStatement(bytes, type, options); + if (isPdf(type, bytes)) { + let text: string; + try { text = await statementPdfText(bytes); } + catch (error) { throw pdfFailure(error, "", options.filename); } + const account = detectPdfStatementAccount(text); + if (!account) { + const message = "PDF is not a recognized Finom or Revolut statement"; + throw new UnrecognizedStatementError(pdfFailure(message, text, options.filename).message); + } + if (options.expectedAccount && account !== options.expectedAccount) { + throw pdfFailure(`PDF contains a ${account} statement, not the selected ${options.expectedAccount} account`, text, options.filename); + } + try { + const parsed = account === "finom" ? parseFinomPdfText(text) : parseRevolutPdfText(text); + if (!parsed.rows.length) throw new ParseError("PDF statement parsed zero transaction rows"); + return { account, layout: `${account} PDF statement`, parsed }; + } catch (error) { + throw pdfFailure(error, text, options.filename); + } + } if (XLSX.test(type)) { for (const account of ["revolut", "finom"] as StatementAccount[]) { try { @@ -158,13 +151,16 @@ async function recognizeEmailStatement(bytes: Buffer, contentType: string, optio if (!(error instanceof ParseError) || !/unrecognized/.test(error.message)) throw error; } } - throw new UnrecognizedStatementError("workbook is not a recognized Finom or Revolut statement export"); + throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: workbook is not a recognized Finom or Revolut statement export`); } const text = bytes.toString("utf8"); const headers = csvHeaders(text); const detected = detectStatementAccount(headers); if (!detected) { - throw new UnrecognizedStatementError(`CSV is not a recognized Finom or Revolut statement export (headers: ${headers.slice(0, 12).join(", ")}${headers.length > 12 ? ", …" : ""})`); + throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: CSV is not a recognized Finom or Revolut statement export (headers: ${headers.slice(0, 12).join(", ")}${headers.length > 12 ? ", …" : ""})`); + } + if (options.expectedAccount && detected.account !== options.expectedAccount) { + throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: CSV contains a ${detected.account} statement, not the selected ${options.expectedAccount} account`); } return { account: detected.account, layout: detected.layout, parsed: parseStatementCsv(text, detected.account, options) }; } @@ -215,7 +211,7 @@ async function importEmailStatement( source: StatementSource, options: { currency?: string; notify?: boolean } & StatementImportOptions = {}, ): Promise { - const recognized = await recognizeEmailStatement(bytes, source.contentType, options); + const recognized = await recognizeStatement(bytes, source.contentType, options); return storeParsedStatement(client, source, recognized.account, recognized.layout, recognized.parsed, options); } diff --git a/backend/src/routes/bookkeepingDocuments.ts b/backend/src/routes/bookkeepingDocuments.ts index 5ba0891cd..d19b2d28b 100644 --- a/backend/src/routes/bookkeepingDocuments.ts +++ b/backend/src/routes/bookkeepingDocuments.ts @@ -42,6 +42,8 @@ const DOCUMENT_TYPES = new Set([ "private-account-statement", "other-evidence", ]); +/** Bank statements may be PDFs or tabular CSV exports; every other document type stays PDF-only. */ +const STATEMENT_CONTENT_TYPES = new Set(["application/pdf", "text/csv"]); export async function handleBookkeepingDocumentRoutes( path: string, @@ -57,8 +59,8 @@ export async function handleBookkeepingDocumentRoutes( const filenameProvided = body?.originalFilename !== undefined; const originalFilename = filenameProvided ? safeUploadFilename(body?.originalFilename) : ""; const requestedContentType = String(body?.contentType || "application/pdf").split(";")[0].trim().toLowerCase(); - if (statement && requestedContentType !== "application/pdf") - return json(400, { error: "Bank statement uploads accept PDF files only" }); + if (statement && !STATEMENT_CONTENT_TYPES.has(requestedContentType)) + return json(400, { error: "Bank statement uploads accept PDF or CSV files only" }); const limit = Number(process.env.BOOKKEEPING_PDF_MAX_BYTES || 20 * 1024 * 1024); if ( !body || @@ -74,7 +76,7 @@ export async function handleBookkeepingDocumentRoutes( (!OPAQUE.test(String(body.accountId || "")) || !MONTH.test(String(body.statementMonth || "")) || (body.contentType !== undefined && - requestedContentType !== "application/pdf"))) || + !STATEMENT_CONTENT_TYPES.has(requestedContentType)))) || (!statement && ((body.accountId !== undefined || body.statementMonth !== undefined) || (body.contentType !== undefined && String(body.contentType) !== "application/pdf"))) @@ -366,7 +368,7 @@ export async function handleBookkeepingDocumentRoutes( ) { const message = !verified.contentOk ? ["bank-statement", "private-account-statement"].includes(String(doc.documentType)) - ? "Bank statement uploads accept PDF files only" + ? "Bank statement uploads accept PDF or CSV files only" : "Evidence uploads accept PDF files only" : "Cleanup deferred"; return json(message === "Cleanup deferred" ? 409 : 400, { error: message }); @@ -388,7 +390,7 @@ export async function handleBookkeepingDocumentRoutes( } const message = !verified.contentOk ? ["bank-statement", "private-account-statement"].includes(String(doc.documentType)) - ? "Bank statement uploads accept PDF files only" + ? "Bank statement uploads accept PDF or CSV files only" : "Evidence uploads accept PDF files only" : "Uploaded object verification failed"; return json(400, { error: message }); diff --git a/backend/src/routes/bookkeepingShared.ts b/backend/src/routes/bookkeepingShared.ts index 2b540a1d8..21369a304 100644 --- a/backend/src/routes/bookkeepingShared.ts +++ b/backend/src/routes/bookkeepingShared.ts @@ -51,7 +51,7 @@ export function publicLink(link: Record) { return safe; } -/** Verify an uploaded evidence or bank-statement object: exact size, digest and PDF signature. */ +/** Verify an uploaded evidence or bank-statement object: exact size, digest and PDF signature (or readable CSV text for statement exports). */ export async function verifyDocumentObject(bucket: string, document: Record) { const object = await s3.send( new GetObjectCommand({ Bucket: bucket, Key: String(document.s3Key) }), @@ -68,15 +68,25 @@ export async function verifyDocumentObject(bucket: string, document: Record Number(process.env.BOOKKEEPING_PDF_MAX_BYTES || 20 * 1024 * 1024) ) throw new Error("object-size-limit"); - if (prefix.length < 5) - prefix = Buffer.concat([prefix, bytes.subarray(0, 5 - prefix.length)]); + if (prefix.length < 512) + prefix = Buffer.concat([prefix, bytes.subarray(0, 512 - prefix.length)]); digest.update(bytes); } const contentType = String(document.contentType || "application/pdf").split(";")[0].trim().toLowerCase(); + const statementDocument = ["bank-statement", "private-account-statement"].includes( + String(document.documentType), + ); + const statementCsv = statementDocument && contentType === "text/csv"; + const looksLikeText = + prefix.length > 0 && + !prefix.includes(0) && + [...prefix].every((byte) => byte >= 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d); return { size, sha256: digest.digest("hex"), - contentOk: contentType === "application/pdf" && prefix.toString("ascii") === "%PDF-", + contentOk: statementCsv + ? looksLikeText + : contentType === "application/pdf" && prefix.subarray(0, 5).toString("ascii") === "%PDF-", versionId: object.VersionId || "", }; } diff --git a/backend/tests/bookkeeping-document-atomic-api.test.ts b/backend/tests/bookkeeping-document-atomic-api.test.ts index 590de007a..b1fb739ad 100644 --- a/backend/tests/bookkeeping-document-atomic-api.test.ts +++ b/backend/tests/bookkeeping-document-atomic-api.test.ts @@ -105,43 +105,58 @@ describe("atomic bookkeeping document API", () => { assert.match(String(signedInputs.at(-1)?.Key), /^documents\//); }); - it("rejects CSV/XLSX bank statement uploads with a PDF-only error at prepare and completion", async () => { - const ownership = { idempotencyKey: "statement-pdf-only-owner" }; - for (const contentType of [ - "text/csv", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - ]) { - const response = await invoke("POST", "/api/bookkeeping/documents/prepare", { - sha256, - byteSize: pdf.length, - documentType: "bank-statement", - accountId: "revolut", - statementMonth: "2026-10", - ...ownership, - sourceRef: `pdf-only-${contentType}`, - contentType, - }); - assert.equal(response.statusCode, 400); - assert.match(JSON.parse(response.body).error, /bank statement uploads accept PDF files only/i); - } + it("accepts text/csv bank statement uploads, still refuses XLSX, and keeps the declared-type check at completion", async () => { + const refused = await invoke("POST", "/api/bookkeeping/documents/prepare", { + sha256, + byteSize: pdf.length, + documentType: "bank-statement", + accountId: "revolut", + statementMonth: "2026-10", + idempotencyKey: "statement-xlsx-refused-owner", + sourceRef: "statement-xlsx-refused-source", + contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + }); + assert.equal(refused.statusCode, 400); + assert.match(JSON.parse(refused.body).error, /bank statement uploads accept PDF or CSV files only/i); - const csv = Buffer.from("Date,Description,Amount\n2026-10-01,Transfer,1.00\n"); + const csv = Buffer.from("Completed date,Counterparty name,Reference,Amount,Balance\n01.10.2026 08:00:00,Sponsor One,202610/01,10.00,110.00\n"); objectBytes = csv; const uploaded = await invoke("POST", "/api/bookkeeping/documents/prepare", { sha256: createHash("sha256").update(csv).digest("hex"), byteSize: csv.length, documentType: "bank-statement", + accountId: "finom", + statementMonth: "2026-10", + originalFilename: "finom-statement.csv", + idempotencyKey: "statement-csv-owner", + sourceRef: "statement-csv-source", + contentType: "text/csv", + }); + assert.equal(uploaded.statusCode, 201, uploaded.body); + const document = JSON.parse(uploaded.body).document; + const completed = await invoke("POST", `/api/bookkeeping/documents/${document.id}/complete`, { + idempotencyKey: "statement-csv-owner", + }); + assert.equal(completed.statusCode, 200, completed.body); + assert.equal(JSON.parse(completed.body).outcome, "created"); + + const mismatchedBytes = Buffer.from("Date,Description,Amount\n2026-10-01,Transfer,1.00\n"); + objectBytes = mismatchedBytes; + const mismatched = await invoke("POST", "/api/bookkeeping/documents/prepare", { + sha256: createHash("sha256").update(mismatchedBytes).digest("hex"), + byteSize: mismatchedBytes.length, + documentType: "bank-statement", accountId: "revolut", statementMonth: "2026-10", idempotencyKey: "statement-pdf-only-body-owner", sourceRef: "statement-pdf-only-body-source", }); - const document = JSON.parse(uploaded.body).document; - const completed = await invoke("POST", `/api/bookkeeping/documents/${document.id}/complete`, { + const mismatchDocument = JSON.parse(mismatched.body).document; + const completedCsvAsPdf = await invoke("POST", `/api/bookkeeping/documents/${mismatchDocument.id}/complete`, { idempotencyKey: "statement-pdf-only-body-owner", }); - assert.equal(completed.statusCode, 400); - assert.match(JSON.parse(completed.body).error, /bank statement uploads accept PDF files only/i); + assert.equal(completedCsvAsPdf.statusCode, 400); + assert.match(JSON.parse(completedCsvAsPdf.body).error, /bank statement uploads accept PDF or CSV files only/i); assert.equal(deletedVersions.at(-1), "version-1"); objectBytes = pdf; }); diff --git a/backend/tests/reconcile.test.ts b/backend/tests/reconcile.test.ts index d6a7aec72..3bcd062f4 100644 --- a/backend/tests/reconcile.test.ts +++ b/backend/tests/reconcile.test.ts @@ -160,52 +160,114 @@ describe('reconcile API', () => { assert.equal(JSON.parse((await call('GET', '/api/reconcile/aliases')).body).items.length, 0); }); - it('rejects CSV and XLSX parse uploads while keeping CSV-backed stored rows readable', async () => { + it('imports a Finom CSV statement with an explicit account, replays idempotently and keeps CSV-backed stored rows readable', async () => { process.env.BOOKKEEPING_DOCUMENTS_BUCKET = 'synthetic-private-bucket'; - let objectBytes = Buffer.from(''); - const fake = { + // The shared fixture shifted to a synthetic year so this test stays isolated from the 2026-10 months. + const csvBytes = Buffer.from(readFileSync(join(__dirname, 'fixtures', 'reconcile', 'finom-main.csv'), 'utf8').replaceAll('2026', '2029')); + let objectBytes = csvBytes; + setBookkeepingStorageForTests({ send: async (command: { constructor: { name: string } }) => { if (command.constructor.name === 'GetObjectCommand') return { Body: Readable.from(objectBytes), VersionId: 'synthetic' }; return {}; }, - }; - setBookkeepingStorageForTests(fake as never); - const inputs = [ - { filename: 'statement.csv', contentType: 'text/csv', bytes: Buffer.from('Date,Description,Amount\n2026-10-02,Transfer,1000.00\n') }, - { filename: 'statement.xlsx', contentType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', bytes: Buffer.from('PK\u0003\u0004synthetic workbook') }, - ]; - for (const [index, input] of inputs.entries()) { - objectBytes = input.bytes; + } as never); + const statementIds: string[] = []; + try { const document = (await putBookkeepingItem(client, 'document', { - documentType: 'bank-statement', status: 'active', contentType: input.contentType, - originalFilename: input.filename, byteSize: objectBytes.length, sha256: `${String(index + 1).repeat(64)}`, s3Key: `documents/parse-test-${index}`, + documentType: 'bank-statement', status: 'active', contentType: 'text/csv', accountId: 'auto', statementMonth: '2029-10', + originalFilename: 'finom_2029-10_main.csv', byteSize: csvBytes.length, sha256: '1'.repeat(64), s3Key: 'documents/finom-csv', })).item; - const parsed = await call('POST', '/api/reconcile/statements/parse', { account: 'revolut', documentId: document.id }); - assert.equal(parsed.statusCode, 422, parsed.body); - assert.match(JSON.parse(parsed.body).error, /statement\.csv|statement\.xlsx/); - assert.match(JSON.parse(parsed.body).error, /PDF files only/); - const imported = await call('POST', '/api/reconcile/statements/import', { documentId: document.id }); - assert.equal(imported.statusCode, 422, imported.body); - assert.match(JSON.parse(imported.body).error, /PDF files only/); - const statements = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items; - assert.ok(!statements.some((statement: Record) => statement.documentId === document.id)); - } - const legacy = await seedStatement({ - account: 'revolut', period: '2030-01', documentId: 'legacy-csv-statement', filename: 'older-export.csv', contentType: 'text/csv', rows: [row({ date: '2030-01-02', amount: '20.00' })], - }); - assert.equal(legacy.statusCode, 201, legacy.body); - const listed = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items; - const retained = listed.find((statement: Record) => statement.documentId === 'legacy-csv-statement'); - assert.equal(retained.rowCount, 1); - assert.equal(retained.incomingCount, 1); - - const notStatement = (await putBookkeepingItem(client, 'document', { - documentType: 'receipt', status: 'active', contentType: 'application/pdf', originalFilename: 'receipt.pdf', byteSize: 9, sha256: 'b'.repeat(64), s3Key: 'documents/receipt', - })).item; - const rejected = await call('POST', '/api/reconcile/statements/parse', { account: 'revolut', documentId: notStatement.id }); - assert.equal(rejected.statusCode, 400); - objectBytes = Buffer.alloc(0); + // A CSV prints no IBAN: without an explicit account the source stays at review and nothing is written. + const review = await call('POST', '/api/reconcile/statements/import', { documentId: document.id, notify: false }); + assert.equal(review.statusCode, 422, review.body); + assert.match(JSON.parse(review.body).error, /stable bank account identifier/); + const beforeReview = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items; + assert.ok(!beforeReview.some((statement: Record) => statement.documentId === document.id)); + + // Parse is read-only: the Finom layout is recognized from the header row alone. + const parsed = await call('POST', '/api/reconcile/statements/parse', { account: 'finom', documentId: document.id }); + assert.equal(parsed.statusCode, 200, parsed.body); + assert.equal(JSON.parse(parsed.body).rowCount, 7); + assert.deepEqual(JSON.parse(parsed.body).balances, [{ currency: 'EUR', amount: '3210.55', asOf: '2029-10-31' }]); + const mismatched = await call('POST', '/api/reconcile/statements/parse', { account: 'revolut', documentId: document.id }); + assert.equal(mismatched.statusCode, 422, mismatched.body); + assert.match(JSON.parse(mismatched.body).error, /CSV contains a finom statement, not the selected revolut account/); + + // Naming the account stands in for the missing header identity; the rows become finom bank rows. + const first = await call('POST', '/api/reconcile/statements/import', { documentId: document.id, account: 'finom', notify: false }); + assert.equal(first.statusCode, 201, first.body); + const firstBody = JSON.parse(first.body); + assert.equal(firstBody.account, 'finom'); + assert.match(firstBody.layout, /Finom statement/); + assert.equal(firstBody.rowCount, 7); + assert.deepEqual(firstBody.balances, [{ currency: 'EUR', amount: '3210.55', asOf: '2029-10-31' }]); + assert.deepEqual(firstBody.statements.map((statement: Record) => statement.period), ['2029-10']); + statementIds.push(firstBody.statements[0].id); + + // Re-uploading the same file replays to the same statement id: identical preview, no change until --replace. + const replay = async (options: Record) => call('POST', '/api/reconcile/statements/import', { documentId: document.id, account: 'finom', notify: false, ...options }); + const preview = await replay({}); + assert.equal(preview.statusCode, 200, preview.body); + assert.equal(JSON.parse(preview.body).committed, false); + assert.equal(JSON.parse(preview.body).preview.requiresConfirmation, true); + assert.match(JSON.parse(preview.body).preview.message, /No differences/); + let listed = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items + .filter((statement: Record) => statement.documentId === document.id); + assert.equal(listed.length, 1); + assert.equal(listed[0].rowCount, 7); + const replaced = await replay({ replace: true, previewDigest: JSON.parse(preview.body).preview.digest }); + assert.equal(replaced.statusCode, 201, replaced.body); + assert.equal(JSON.parse(replaced.body).committed, true); + assert.deepEqual(JSON.parse(replaced.body).statements.map((statement: Record) => statement.id), statementIds.slice(-1)); + + const balances = JSON.parse((await invoke('GET', '/api/reconcile/balances', undefined, AUTH, { account: 'finom' })).body); + assert.ok(balances.items.some((item: Record) => item.amount === '3210.55' && item.asOf === '2029-10-31')); + + // Overlapping rows from a second export are counted once: date+amount+counterparty identity, earliest upload wins. + const overlap = [ + 'Completed date,Counterparty name,Reference,Amount,Balance', + '05.10.2029 08:00:00,Sponsor Three Ltd,202910/03,1 000.00,3 000.00', + '06.10.2029 09:00:00,Sponsor Four GmbH,202910/04,500.00,3 500.00', + ].join('\n'); + objectBytes = Buffer.from(overlap); + const second = (await putBookkeepingItem(client, 'document', { + documentType: 'bank-statement', status: 'active', contentType: 'text/csv', accountId: 'auto', statementMonth: '2029-10', + originalFilename: 'finom_2029-10_extra.csv', byteSize: objectBytes.length, sha256: '2'.repeat(64), s3Key: 'documents/finom-csv-2', + })).item; + const secondImport = await call('POST', '/api/reconcile/statements/import', { documentId: second.id, account: 'finom', notify: false }); + assert.equal(secondImport.statusCode, 201, secondImport.body); + statementIds.push(JSON.parse(secondImport.body).statements[0].id); + const month = JSON.parse((await invoke('GET', '/api/reconcile/months/2029-10', undefined, AUTH)).body); + assert.equal(month.rows.filter((row: Record) => row.counterparty === 'Sponsor Three Ltd').length, 1); + assert.equal(month.rows.filter((row: Record) => row.counterparty === 'Sponsor Four GmbH').length, 1); + + // A generic CSV matches no bank layout and is refused before anything is stored. + objectBytes = Buffer.from('Date,Description,Amount\n2029-10-02,Transfer,1000.00\n'); + const unknown = (await putBookkeepingItem(client, 'document', { + documentType: 'bank-statement', status: 'active', contentType: 'text/csv', accountId: 'auto', statementMonth: '2029-10', + originalFilename: 'notes.csv', byteSize: objectBytes.length, sha256: '3'.repeat(64), s3Key: 'documents/notes-csv', + })).item; + const refused = await call('POST', '/api/reconcile/statements/import', { documentId: unknown.id, account: 'finom', notify: false }); + assert.equal(refused.statusCode, 422, refused.body); + assert.match(JSON.parse(refused.body).error, /notes\.csv/); + assert.match(JSON.parse(refused.body).error, /CSV is not a recognized/); + const afterRefusal = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items; + assert.ok(!afterRefusal.some((statement: Record) => statement.documentId === unknown.id)); + + // Legacy CSV-backed statements stay readable without byte re-verification. + const legacy = await seedStatement({ + account: 'revolut', period: '2030-01', documentId: 'legacy-csv-statement', filename: 'older-export.csv', contentType: 'text/csv', rows: [row({ date: '2030-01-02', amount: '20.00' })], + }); + assert.equal(legacy.statusCode, 201, legacy.body); + const retained = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items + .find((statement: Record) => statement.documentId === 'legacy-csv-statement'); + assert.equal(retained.rowCount, 1); + assert.equal(retained.incomingCount, 1); + } finally { + for (const id of statementIds) await call('DELETE', `/api/reconcile/statements/${id}`); + delete process.env.BOOKKEEPING_DOCUMENTS_BUCKET; + } }); it('rejects malformed Finom/Revolut rows, invalid dates, zero-row PDFs and balance mismatches atomically', async () => { @@ -572,18 +634,21 @@ describe('reconcile API', () => { assert.deepEqual(summary.items.map((item: { period: string }) => item.period), ['2026-11', '2026-12']); }); - it('rejects a CSV from the statement import route before writing rows', async () => { + it('keeps a Finom CSV without an account, or with the wrong account, from writing rows', async () => { process.env.BOOKKEEPING_DOCUMENTS_BUCKET = 'synthetic-private-bucket'; - let objectBytes = readFileSync(join(__dirname, 'fixtures', 'reconcile', 'finom-main.csv')); + const objectBytes = readFileSync(join(__dirname, 'fixtures', 'reconcile', 'finom-main.csv')); setBookkeepingStorageForTests({ send: async (command: { constructor: { name: string } }) => command.constructor.name === 'GetObjectCommand' ? { Body: Readable.from(objectBytes) } : {} } as never); const document = (await putBookkeepingItem(client, 'document', { documentType: 'bank-statement', status: 'active', contentType: 'text/csv', accountId: 'auto', statementMonth: '2026-10', originalFilename: 'finom_2026-10_main.csv', byteSize: objectBytes.length, sha256: 'f'.repeat(64), s3Key: 'documents/finom-auto', })).item; - const imported = await call('POST', '/api/reconcile/statements/import', { documentId: document.id }); - assert.equal(imported.statusCode, 422, imported.body); - assert.match(JSON.parse(imported.body).error, /finom_2026-10_main\.csv/); - assert.match(JSON.parse(imported.body).error, /PDF files only/); + const accountless = await call('POST', '/api/reconcile/statements/import', { documentId: document.id }); + assert.equal(accountless.statusCode, 422, accountless.body); + assert.match(JSON.parse(accountless.body).error, /stable bank account identifier/); + const wrongAccount = await call('POST', '/api/reconcile/statements/import', { documentId: document.id, account: 'revolut' }); + assert.equal(wrongAccount.statusCode, 422, wrongAccount.body); + assert.match(JSON.parse(wrongAccount.body).error, /finom_2026-10_main\.csv/); + assert.match(JSON.parse(wrongAccount.body).error, /CSV contains a finom statement, not the selected revolut account/); const statements = JSON.parse((await call('GET', '/api/reconcile/statements')).body).items; assert.ok(!statements.some((statement: Record) => statement.documentId === document.id)); }); diff --git a/cli/src/commands/reconcile.mjs b/cli/src/commands/reconcile.mjs index e8b8c68a1..5b5cfbb42 100644 --- a/cli/src/commands/reconcile.mjs +++ b/cli/src/commands/reconcile.mjs @@ -6,7 +6,7 @@ import { dirname, join, relative, resolve, sep, posix, win32 } from "node:path"; import { UsageError } from "../errors.mjs"; import { enc, optionalString, positional } from "../format.mjs"; import { apiFor } from "../session.mjs"; -import { contentTypeFor, requirePdfBytes, storeDocument } from "../documentUpload.mjs"; +import { contentTypeFor, requireStatementBytes, storeDocument } from "../documentUpload.mjs"; import { attachThankYouResult, printThankYouHint } from "../thank-you.mjs"; export const summary = "Month-end reconciliation: bank statements, matching, payment confirm"; @@ -14,7 +14,7 @@ export const summary = "Month-end reconciliation: bank statements, matching, pay export const help = `dataops reconcile - month-end reconciliation (the Finance views' command line) Usage - dataops reconcile upload --file statement.pdf [--account finom|revolut] [--period YYYY-MM] [--quiet] [--preview|--replace] [--json] + dataops reconcile upload --file statement.pdf|statement.csv [--account finom|revolut] [--period YYYY-MM] [--quiet] [--preview|--replace] [--json] Finom and Revolut statements need no flags: the bank comes from the file, and its rows are stored as one statement per month, then each month is reconciled and summarised in a finance notification @@ -23,6 +23,9 @@ Usage preview without changing active data. --replace explicitly confirms the current preview; --preview/--dry-run only previews. Both banks preserve original PDF history and use the printed account, wallet and month. + CSV exports (the Finom statement CSV) are accepted too; they carry no + IBAN, so pass --account to import them — without it the source is kept + for review and nothing changes. dataops reconcile import-document [--period YYYY-MM] [--quiet] [--preview|--replace] [--json] Parse a bank-statement document DataOps already stores (bank detected from its content) into per-month statements. The same preview and explicit @@ -145,11 +148,12 @@ function printStatementImport(result, io) { } export async function upload(args, io) { - if (!args.file) throw new UsageError("dataops reconcile upload --file statement.pdf [--account finom|revolut] [--period YYYY-MM] [--preview|--replace] [--json]"); + if (!args.file) throw new UsageError("dataops reconcile upload --file statement.pdf|statement.csv [--account finom|revolut] [--period YYYY-MM] [--preview|--replace] [--json]"); if (args.account && !ACCOUNTS.includes(String(args.account))) throw new UsageError(`--account must be one of ${ACCOUNTS.join(", ")}`); if (args.period) periodOr(args); - contentTypeFor(args.file); - requirePdfBytes(args.file, await readFile(String(args.file))); + const contentType = contentTypeFor(args.file); + const bytes = await readFile(String(args.file)); + requireStatementBytes(args.file, bytes, contentType); const api = apiFor(args); const stored = await storeDocument(api, args.file, { documentType: "bank-statement", accountId: args.account || "auto", statementMonth: args.period || new Date().toISOString().slice(0, 7), sourceRef: "reconcile:upload" }); const result = await importStoredStatement(api, stored.documentId, args); @@ -792,7 +796,7 @@ export async function alias(args, io) { } export const commands = { - upload, "import-document": importDocument, parse, balances, statements, statement, report, confirm, aliases, alias, + upload, "upload-statement": upload, "import-document": importDocument, parse, balances, statements, statement, report, confirm, aliases, alias, month, months, matches, "auto-match": autoMatch, match, unmatch, "attach-proof": attachProof, "attach-doc": attachDoc, "void-ledger": voidLedger, "restore-ledger": restoreLedger, "create-ledger": createLedger, note, disposition, except, "clear-except": clearExcept, "confirm-exceptions": confirmExceptions, close, reopen, package: packageMonth, config, }; diff --git a/cli/src/documentUpload.mjs b/cli/src/documentUpload.mjs index 3121d1fbf..c570b7ce0 100644 --- a/cli/src/documentUpload.mjs +++ b/cli/src/documentUpload.mjs @@ -5,12 +5,13 @@ import { enc } from "./format.mjs"; const CONTENT_TYPES = { ".pdf": "application/pdf", + ".csv": "text/csv", }; function contentTypeFor(file) { const suffix = String(file).toLowerCase().match(/\.[a-z0-9]+$/)?.[0] || ""; const contentType = CONTENT_TYPES[suffix]; - if (!contentType) throw new UsageError("Document uploads accept PDF files only (.pdf)"); + if (!contentType) throw new UsageError("Document uploads accept PDF files only (.pdf) — bank statements also accept CSV (.csv)"); return contentType; } @@ -20,19 +21,28 @@ function requirePdfBytes(file, bytes) { } } -export { contentTypeFor, requirePdfBytes, originalFilename }; +/** Bank statements may be CSV exports: readable text, not a PDF or binary file renamed to .csv. */ +function requireStatementBytes(file, bytes, contentType) { + if (contentType === "application/pdf") return requirePdfBytes(file, bytes); + if (bytes.subarray(0, 5).toString("ascii") === "%PDF-" || bytes.includes(0)) { + throw new UsageError(`${file}: not a readable CSV statement export`); + } +} + +export { contentTypeFor, requirePdfBytes, requireStatementBytes, originalFilename }; function originalFilename(file) { return String(file).split(/[\\/]/).at(-1) || "document.pdf"; } -/** Upload one PDF through prepare/PUT/complete; a file already stored comes back as reused. */ +/** Upload one PDF (or a bank-statement CSV) through prepare/PUT/complete; a file already stored comes back as reused. */ export async function storeDocument(api, file, { documentType, accountId, statementMonth, sourceRef }) { const contentType = contentTypeFor(file); const statement = ["bank-statement", "private-account-statement"].includes(documentType); if (!statement && contentType !== "application/pdf") throw new UsageError("invoices, receipts and proof screenshots must be PDF files"); const bytes = await readFile(String(file)); - requirePdfBytes(file, bytes); + if (statement) requireStatementBytes(file, bytes, contentType); + else requirePdfBytes(file, bytes); const sha256 = createHash("sha256").update(bytes).digest("hex"); const idempotencyKey = `${documentType}-${sha256}`.slice(0, 120); let prepared = await api.post("/api/bookkeeping/documents/prepare", { diff --git a/cli/test/reconcile.test.mjs b/cli/test/reconcile.test.mjs index d49a0478f..9ffe17acf 100644 --- a/cli/test/reconcile.test.mjs +++ b/cli/test/reconcile.test.mjs @@ -102,21 +102,54 @@ test("PDF upload stores the raw statement, parses it and saves rows for matching await rm(directory, { recursive: true, force: true }); }); -test("upload rejects CSV and XLSX locally with a non-zero usage exit code", async () => { +test("upload stores a Finom CSV statement and imports it under the named account", async () => { + const directory = await mkdtemp(join(tmpdir(), "dataops-reconcile-csv-")); + const path = join(directory, "finom-statement.csv"); + await writeFile(path, [ + "Completed date,Counterparty name,Reference,Amount,Balance", + "05.10.2026 08:00:00,Sponsor Three Ltd,202610/03,1 000.00,3 000.00", + ].join("\n")); + try { + await withFakePortal([ + ["/documents/prepare", (request) => { + const body = JSON.parse(request.options.body); + assert.equal(body.documentType, "bank-statement"); + assert.equal(body.accountId, "finom"); + assert.equal(body.statementMonth, "2026-10"); + assert.equal(body.originalFilename, "finom-statement.csv"); + assert.equal(body.contentType, "text/csv"); + assert.match(body.idempotencyKey, /^bank-statement-[0-9a-f]{64}$/); + return { outcome: "existing", document: { id: "finom-csv-doc" } }; + }], + ["/reconcile/statements/import", (request) => { + const body = JSON.parse(request.options.body); + assert.equal(body.documentId, "finom-csv-doc"); assert.equal(body.account, "finom"); assert.equal(body.rows, undefined, "rows are parsed by the server, never supplied by the CLI"); + return { committed: true, statements: [{ id: "stmt-finom-csv", period: "2026-10", rowCount: 1 }], balances: [], summaries: [] }; + }], + ], async () => { + assert.equal(commands["upload-statement"], upload, "upload-statement is the same upload flow under its own name"); + const result = await upload({ ...args, file: path, account: "finom", period: "2026-10" }, {}); + assert.equal(result.documentId, "finom-csv-doc"); + assert.equal(result.statements[0].id, "stmt-finom-csv"); + }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("upload refuses XLSX, PDF-mislabeled and binary CSV statements with a usage exit code", async () => { const directory = await mkdtemp(join(tmpdir(), "dataops-reconcile-pdf-only-")); try { - for (const extension of ["csv", "xlsx"]) { - const path = join(directory, `statement.${extension}`); - await writeFile(path, "synthetic bank export"); - await assert.rejects( - upload({ ...args, file: path, account: "revolut", period: "2026-10" }, {}), - (error) => { - assert.match(error.message, /PDF files only/); - assert.equal(exitCodeFor(error), EXIT.usage); - return true; - }, - ); - } + const xlsx = join(directory, "statement.xlsx"); + await writeFile(xlsx, "synthetic bank export"); + await assert.rejects( + upload({ ...args, file: xlsx, account: "revolut", period: "2026-10" }, {}), + (error) => { + assert.match(error.message, /PDF files only/); + assert.equal(exitCodeFor(error), EXIT.usage); + return true; + }, + ); const invalidPdf = join(directory, "not-a-pdf.pdf"); await writeFile(invalidPdf, "Date,Amount\n2026-10-01,1.00\n"); await assert.rejects( @@ -127,6 +160,28 @@ test("upload rejects CSV and XLSX locally with a non-zero usage exit code", asyn return true; }, ); + const binaryCsv = join(directory, "renamed-binary.csv"); + await writeFile(binaryCsv, Buffer.from("PK\u0003\u0004synthetic workbook with NUL \u0000 bytes")); + await assert.rejects( + upload({ ...args, file: binaryCsv, account: "finom", period: "2026-10" }, {}), + (error) => { + assert.match(error.message, /not a readable CSV statement export/); + assert.equal(exitCodeFor(error), EXIT.usage); + return true; + }, + ); + const receiptCsv = join(directory, "receipt.csv"); + await writeFile(receiptCsv, "Invoice,Total\n2026-101,12.00\n"); + await withFakePortal([], async () => { + await assert.rejects( + attachDoc({ ...args, _: ["ledger:ledger-x", receiptCsv], period: "2026-10" }, {}), + (error) => { + assert.match(error.message, /invoices, receipts and proof screenshots must be PDF files/); + assert.equal(exitCodeFor(error), EXIT.usage); + return true; + }, + ); + }); } finally { await rm(directory, { recursive: true, force: true }); }