Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions backend/src/reconcile/statementImport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import { coverageGaps } from "./match";

export class StatementIdentityReviewError extends Error {
statusCode = 422;
constructor(readonly account: StatementAccount) { super("Statement header has no stable bank account identifier; an IBAN or account number is required before import. Source retained for review; no active statement changed."); }
constructor(readonly account: StatementAccount, message?: string) { super(message || "Statement header has no stable bank account identifier; an IBAN or account number is required before import. Source retained for review; no active statement changed."); }
}

export type StatementImportOptions = { preview?: boolean; replace?: boolean; previewDigest?: string; expectedAccount?: StatementAccount; expectedPeriod?: string; by?: string };
Expand Down Expand Up @@ -70,7 +70,10 @@ export async function prepareStatementImport(client: DynamoDBDocumentClient, sou
if (options.expectedPeriod && !periods.includes(options.expectedPeriod)) throw new Error("Statement header/rows do not cover the selected month");
if (periods.length > 24) throw new Error("Import at most 24 months in one statement batch");
const missingIdentity = [...parsed.rows, ...parsed.balances].some((row) => !parsed.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]);
if (missingIdentity) throw new StatementIdentityReviewError(account);
// Tabular exports (Finom CSV) print no IBAN. They commit only when the
// operator explicitly selected the account, which stands in for the header
// identity; otherwise the source waits for review, as the email intake does.
if (missingIdentity && !options.expectedAccount) throw new StatementIdentityReviewError(account);
const history = await listStatementHistory(client);
const months = await Promise.all(periods.map((period) => getMonth(client, period)));
const skippedClosedMonths = months.filter((month) => month.state === "closed").map((month) => month.period);
Expand Down Expand Up @@ -106,7 +109,11 @@ export async function prepareStatementImport(client: DynamoDBDocumentClient, sou
const candidate: StatementRecord = { id, recordType: "reconcile-statement", accountId: account, period, ...source, rows, balances, ...bounds,
bankAccounts: parsed.bankAccounts, ...(parsed.issueDate ? { issueDate: parsed.issueDate } : {}), ...(parsed.openingBalances ? { openingBalances: parsed.openingBalances.filter((balance) => balance.asOf.startsWith(period)) } : {}),
warnings: parsed.warnings, createdAt: previous?.createdAt || now, updatedAt: now };
if (verified.some((record) => record.accountId === account && overlaps(record, candidate) && [...record.rows, ...(record.balances || [])].some((row) => !record.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]))) throw new StatementIdentityReviewError(account);
// Only a source with header identity may not coexist with an unverified one; two
// operator-asserted (identity-less) sources of the same account overlap harmlessly.
const candidateVerified = [...rows, ...balances].every((row) => parsed.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`]);
if (candidateVerified && verified.some((record) => record.accountId === account && overlaps(record, candidate) && [...record.rows, ...(record.balances || [])].some((row) => !record.bankAccounts?.[`${row.wallet || "main"}|${row.currency}`])))
throw new StatementIdentityReviewError(account, "An overlapping statement source of this account carries no bank account identifier; delete or replace that source before importing a statement with verified identity. No active statement changed.");
const keys = [...new Set([...rows, ...balances].map((row) => walletKey(candidate, row)))].sort();
const existing = active.flatMap((record) => {
if (record.accountId !== account || !overlaps(record, candidate)) return [];
Expand Down
68 changes: 32 additions & 36 deletions backend/src/reconcile/statementIntake.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,42 +114,35 @@ function pdfFailure(error: unknown, text: string, filename?: string): ParseError
return new ParseError(`${safeFilename(filename)}: line ${source.number}, source text "${excerpt}": ${message}`);
}

/** Recognize and completely parse a supported bank PDF before any statement rows are stored. */
export async function recognizeStatement(bytes: Buffer, contentType: string, options: { currency?: string; expectedAccount?: StatementAccount; filename?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> {
const type = contentType.split(";")[0].trim().toLowerCase();
if (!isPdf(type, bytes)) {
throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: bank statement uploads accept PDF files only`);
}

let text: string;
try { text = await statementPdfText(bytes); }
catch (error) { throw pdfFailure(error, "", options.filename); }
const account = detectPdfStatementAccount(text);
if (!account) {
const message = "PDF is not a recognized Finom or Revolut statement";
throw new UnrecognizedStatementError(pdfFailure(message, text, options.filename).message);
}
if (options.expectedAccount && account !== options.expectedAccount) {
throw pdfFailure(`PDF contains a ${account} statement, not the selected ${options.expectedAccount} account`, text, options.filename);
}

try {
const parsed = account === "finom" ? parseFinomPdfText(text) : parseRevolutPdfText(text);
if (!parsed.rows.length) throw new ParseError("PDF statement parsed zero transaction rows");
return { account, layout: `${account} PDF statement`, parsed };
} catch (error) {
throw pdfFailure(error, text, options.filename);
}
}

/**
* Email document intake keeps its existing tabular attachment handling.
* The explicit reconcile upload API, CLI and portal use recognizeStatement,
* which accepts PDFs only.
* Recognize and completely parse a supported bank statement before any rows
* are stored. PDFs are read through their bank's fixed layout; tabular
* exports (CSV, and XLSX from the email intake) are recognized from their
* header row against the same fixed layouts. Used by the explicit reconcile
* upload API, CLI, portal and the email intake.
*/
async function recognizeEmailStatement(bytes: Buffer, contentType: string, options: { currency?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> {
export async function recognizeStatement(bytes: Buffer, contentType: string, options: { currency?: string; expectedAccount?: StatementAccount; filename?: string } = {}): Promise<{ account: StatementAccount; layout: string; parsed: ParsedStatement }> {
const type = contentType.split(";")[0].trim().toLowerCase();
if (isPdf(type, bytes)) return recognizeStatement(bytes, type, options);
if (isPdf(type, bytes)) {
let text: string;
try { text = await statementPdfText(bytes); }
catch (error) { throw pdfFailure(error, "", options.filename); }
const account = detectPdfStatementAccount(text);
if (!account) {
const message = "PDF is not a recognized Finom or Revolut statement";
throw new UnrecognizedStatementError(pdfFailure(message, text, options.filename).message);
}
if (options.expectedAccount && account !== options.expectedAccount) {
throw pdfFailure(`PDF contains a ${account} statement, not the selected ${options.expectedAccount} account`, text, options.filename);
}
try {
const parsed = account === "finom" ? parseFinomPdfText(text) : parseRevolutPdfText(text);
if (!parsed.rows.length) throw new ParseError("PDF statement parsed zero transaction rows");
return { account, layout: `${account} PDF statement`, parsed };
} catch (error) {
throw pdfFailure(error, text, options.filename);
}
}
if (XLSX.test(type)) {
for (const account of ["revolut", "finom"] as StatementAccount[]) {
try {
Expand All @@ -158,13 +151,16 @@ async function recognizeEmailStatement(bytes: Buffer, contentType: string, optio
if (!(error instanceof ParseError) || !/unrecognized/.test(error.message)) throw error;
}
}
throw new UnrecognizedStatementError("workbook is not a recognized Finom or Revolut statement export");
throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: workbook is not a recognized Finom or Revolut statement export`);
}
const text = bytes.toString("utf8");
const headers = csvHeaders(text);
const detected = detectStatementAccount(headers);
if (!detected) {
throw new UnrecognizedStatementError(`CSV is not a recognized Finom or Revolut statement export (headers: ${headers.slice(0, 12).join(", ")}${headers.length > 12 ? ", …" : ""})`);
throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: CSV is not a recognized Finom or Revolut statement export (headers: ${headers.slice(0, 12).join(", ")}${headers.length > 12 ? ", …" : ""})`);
}
if (options.expectedAccount && detected.account !== options.expectedAccount) {
throw new UnrecognizedStatementError(`${safeFilename(options.filename)}: CSV contains a ${detected.account} statement, not the selected ${options.expectedAccount} account`);
}
return { account: detected.account, layout: detected.layout, parsed: parseStatementCsv(text, detected.account, options) };
}
Expand Down Expand Up @@ -215,7 +211,7 @@ async function importEmailStatement(
source: StatementSource,
options: { currency?: string; notify?: boolean } & StatementImportOptions = {},
): Promise<ImportedStatement> {
const recognized = await recognizeEmailStatement(bytes, source.contentType, options);
const recognized = await recognizeStatement(bytes, source.contentType, options);
return storeParsedStatement(client, source, recognized.account, recognized.layout, recognized.parsed, options);
}

Expand Down
12 changes: 7 additions & 5 deletions backend/src/routes/bookkeepingDocuments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ const DOCUMENT_TYPES = new Set([
"private-account-statement",
"other-evidence",
]);
/** Bank statements may be PDFs or tabular CSV exports; every other document type stays PDF-only. */
const STATEMENT_CONTENT_TYPES = new Set(["application/pdf", "text/csv"]);

export async function handleBookkeepingDocumentRoutes(
path: string,
Expand All @@ -57,8 +59,8 @@ export async function handleBookkeepingDocumentRoutes(
const filenameProvided = body?.originalFilename !== undefined;
const originalFilename = filenameProvided ? safeUploadFilename(body?.originalFilename) : "";
const requestedContentType = String(body?.contentType || "application/pdf").split(";")[0].trim().toLowerCase();
if (statement && requestedContentType !== "application/pdf")
return json(400, { error: "Bank statement uploads accept PDF files only" });
if (statement && !STATEMENT_CONTENT_TYPES.has(requestedContentType))
return json(400, { error: "Bank statement uploads accept PDF or CSV files only" });
const limit = Number(process.env.BOOKKEEPING_PDF_MAX_BYTES || 20 * 1024 * 1024);
if (
!body ||
Expand All @@ -74,7 +76,7 @@ export async function handleBookkeepingDocumentRoutes(
(!OPAQUE.test(String(body.accountId || "")) ||
!MONTH.test(String(body.statementMonth || "")) ||
(body.contentType !== undefined &&
requestedContentType !== "application/pdf"))) ||
!STATEMENT_CONTENT_TYPES.has(requestedContentType)))) ||
(!statement &&
((body.accountId !== undefined || body.statementMonth !== undefined) ||
(body.contentType !== undefined && String(body.contentType) !== "application/pdf")))
Expand Down Expand Up @@ -366,7 +368,7 @@ export async function handleBookkeepingDocumentRoutes(
) {
const message = !verified.contentOk
? ["bank-statement", "private-account-statement"].includes(String(doc.documentType))
? "Bank statement uploads accept PDF files only"
? "Bank statement uploads accept PDF or CSV files only"
: "Evidence uploads accept PDF files only"
: "Cleanup deferred";
return json(message === "Cleanup deferred" ? 409 : 400, { error: message });
Expand All @@ -388,7 +390,7 @@ export async function handleBookkeepingDocumentRoutes(
}
const message = !verified.contentOk
? ["bank-statement", "private-account-statement"].includes(String(doc.documentType))
? "Bank statement uploads accept PDF files only"
? "Bank statement uploads accept PDF or CSV files only"
: "Evidence uploads accept PDF files only"
: "Uploaded object verification failed";
return json(400, { error: message });
Expand Down
18 changes: 14 additions & 4 deletions backend/src/routes/bookkeepingShared.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ export function publicLink(link: Record<string, unknown>) {
return safe;
}

/** Verify an uploaded evidence or bank-statement object: exact size, digest and PDF signature. */
/** Verify an uploaded evidence or bank-statement object: exact size, digest and PDF signature (or readable CSV text for statement exports). */
export async function verifyDocumentObject(bucket: string, document: Record<string, unknown>) {
const object = await s3.send(
new GetObjectCommand({ Bucket: bucket, Key: String(document.s3Key) }),
Expand All @@ -68,15 +68,25 @@ export async function verifyDocumentObject(bucket: string, document: Record<stri
size > Number(process.env.BOOKKEEPING_PDF_MAX_BYTES || 20 * 1024 * 1024)
)
throw new Error("object-size-limit");
if (prefix.length < 5)
prefix = Buffer.concat([prefix, bytes.subarray(0, 5 - prefix.length)]);
if (prefix.length < 512)
prefix = Buffer.concat([prefix, bytes.subarray(0, 512 - prefix.length)]);
digest.update(bytes);
}
const contentType = String(document.contentType || "application/pdf").split(";")[0].trim().toLowerCase();
const statementDocument = ["bank-statement", "private-account-statement"].includes(
String(document.documentType),
);
const statementCsv = statementDocument && contentType === "text/csv";
const looksLikeText =
prefix.length > 0 &&
!prefix.includes(0) &&
[...prefix].every((byte) => byte >= 0x20 || byte === 0x09 || byte === 0x0a || byte === 0x0d);
return {
size,
sha256: digest.digest("hex"),
contentOk: contentType === "application/pdf" && prefix.toString("ascii") === "%PDF-",
contentOk: statementCsv
? looksLikeText
: contentType === "application/pdf" && prefix.subarray(0, 5).toString("ascii") === "%PDF-",
versionId: object.VersionId || "",
};
}
Expand Down
63 changes: 39 additions & 24 deletions backend/tests/bookkeeping-document-atomic-api.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -105,43 +105,58 @@ describe("atomic bookkeeping document API", () => {
assert.match(String(signedInputs.at(-1)?.Key), /^documents\//);
});

it("rejects CSV/XLSX bank statement uploads with a PDF-only error at prepare and completion", async () => {
const ownership = { idempotencyKey: "statement-pdf-only-owner" };
for (const contentType of [
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
]) {
const response = await invoke("POST", "/api/bookkeeping/documents/prepare", {
sha256,
byteSize: pdf.length,
documentType: "bank-statement",
accountId: "revolut",
statementMonth: "2026-10",
...ownership,
sourceRef: `pdf-only-${contentType}`,
contentType,
});
assert.equal(response.statusCode, 400);
assert.match(JSON.parse(response.body).error, /bank statement uploads accept PDF files only/i);
}
it("accepts text/csv bank statement uploads, still refuses XLSX, and keeps the declared-type check at completion", async () => {
const refused = await invoke("POST", "/api/bookkeeping/documents/prepare", {
sha256,
byteSize: pdf.length,
documentType: "bank-statement",
accountId: "revolut",
statementMonth: "2026-10",
idempotencyKey: "statement-xlsx-refused-owner",
sourceRef: "statement-xlsx-refused-source",
contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
});
assert.equal(refused.statusCode, 400);
assert.match(JSON.parse(refused.body).error, /bank statement uploads accept PDF or CSV files only/i);

const csv = Buffer.from("Date,Description,Amount\n2026-10-01,Transfer,1.00\n");
const csv = Buffer.from("Completed date,Counterparty name,Reference,Amount,Balance\n01.10.2026 08:00:00,Sponsor One,202610/01,10.00,110.00\n");
objectBytes = csv;
const uploaded = await invoke("POST", "/api/bookkeeping/documents/prepare", {
sha256: createHash("sha256").update(csv).digest("hex"),
byteSize: csv.length,
documentType: "bank-statement",
accountId: "finom",
statementMonth: "2026-10",
originalFilename: "finom-statement.csv",
idempotencyKey: "statement-csv-owner",
sourceRef: "statement-csv-source",
contentType: "text/csv",
});
assert.equal(uploaded.statusCode, 201, uploaded.body);
const document = JSON.parse(uploaded.body).document;
const completed = await invoke("POST", `/api/bookkeeping/documents/${document.id}/complete`, {
idempotencyKey: "statement-csv-owner",
});
assert.equal(completed.statusCode, 200, completed.body);
assert.equal(JSON.parse(completed.body).outcome, "created");

const mismatchedBytes = Buffer.from("Date,Description,Amount\n2026-10-01,Transfer,1.00\n");
objectBytes = mismatchedBytes;
const mismatched = await invoke("POST", "/api/bookkeeping/documents/prepare", {
sha256: createHash("sha256").update(mismatchedBytes).digest("hex"),
byteSize: mismatchedBytes.length,
documentType: "bank-statement",
accountId: "revolut",
statementMonth: "2026-10",
idempotencyKey: "statement-pdf-only-body-owner",
sourceRef: "statement-pdf-only-body-source",
});
const document = JSON.parse(uploaded.body).document;
const completed = await invoke("POST", `/api/bookkeeping/documents/${document.id}/complete`, {
const mismatchDocument = JSON.parse(mismatched.body).document;
const completedCsvAsPdf = await invoke("POST", `/api/bookkeeping/documents/${mismatchDocument.id}/complete`, {
idempotencyKey: "statement-pdf-only-body-owner",
});
assert.equal(completed.statusCode, 400);
assert.match(JSON.parse(completed.body).error, /bank statement uploads accept PDF files only/i);
assert.equal(completedCsvAsPdf.statusCode, 400);
assert.match(JSON.parse(completedCsvAsPdf.body).error, /bank statement uploads accept PDF or CSV files only/i);
assert.equal(deletedVersions.at(-1), "version-1");
objectBytes = pdf;
});
Expand Down
Loading