From 61ec79e38a0593ea939450bef4b5b8f5b5df6925 Mon Sep 17 00:00:00 2001 From: Alessandro Franceschi Date: Thu, 24 Sep 2026 18:56:14 +0200 Subject: [PATCH] fix(puppetdb): reset circuit breaker failures on success while closed LifetimeCount accumulated for the process lifetime while closed, so sparse failures on long-running instances tripped the breaker even when PuppetDB was healthy. Clear the streak on success, and log grouping query errors. Closes #59 Co-authored-by: Cursor --- .../integrations/puppetdb/CircuitBreaker.ts | 3 + .../integrations/puppetdb/PuppetDBService.ts | 10 ++- .../unit/integrations/CircuitBreaker.test.ts | 79 +++++++++++++++++++ 3 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 backend/test/unit/integrations/CircuitBreaker.test.ts diff --git a/backend/src/integrations/puppetdb/CircuitBreaker.ts b/backend/src/integrations/puppetdb/CircuitBreaker.ts index 42fff2a7..2b976d92 100644 --- a/backend/src/integrations/puppetdb/CircuitBreaker.ts +++ b/backend/src/integrations/puppetdb/CircuitBreaker.ts @@ -165,6 +165,9 @@ export class CircuitBreaker { // Service has recovered, close the circuit this.reset(); this.transitionTo("closed"); + } else if (this.state === "closed") { + // Only consecutive failures trip the breaker; a success clears the streak + this.failureCount = 0; } } diff --git a/backend/src/integrations/puppetdb/PuppetDBService.ts b/backend/src/integrations/puppetdb/PuppetDBService.ts index 9654b7d2..587eedc7 100644 --- a/backend/src/integrations/puppetdb/PuppetDBService.ts +++ b/backend/src/integrations/puppetdb/PuppetDBService.ts @@ -656,8 +656,9 @@ export class PuppetDBService const envGroups = this.createEnvironmentGroups(envResult as PuppetDBNode[]); groups.push(...envGroups); } - } catch { - this.log("Failed to query nodes for environment grouping", "warn"); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + this.log(`Failed to query nodes for environment grouping: ${detail}`, "warn"); } // Query 2: Group by OS family (from facts) @@ -673,8 +674,9 @@ export class PuppetDBService const osGroups = this.createOSFamilyGroups(osResult as { certname: string; "os.family": string }[]); groups.push(...osGroups); } - } catch { - this.log("Failed to query nodes for OS family grouping", "warn"); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + this.log(`Failed to query nodes for OS family grouping: ${detail}`, "warn"); } // Cache the result diff --git a/backend/test/unit/integrations/CircuitBreaker.test.ts b/backend/test/unit/integrations/CircuitBreaker.test.ts new file mode 100644 index 00000000..d4467162 --- /dev/null +++ b/backend/test/unit/integrations/CircuitBreaker.test.ts @@ -0,0 +1,79 @@ +/** + * Unit tests for CircuitBreaker + * + * Covers consecutive-failure semantics while closed (issue #59). + */ + +import { describe, it, expect, beforeEach } from "vitest"; +import { + CircuitBreaker, + CircuitBreakerOpenError, +} from "../../../src/integrations/puppetdb/CircuitBreaker"; + +describe("CircuitBreaker", () => { + let breaker: CircuitBreaker; + + beforeEach(() => { + breaker = new CircuitBreaker({ + failureThreshold: 3, + resetTimeout: 1000, + }); + }); + + describe("failure counting while closed", () => { + it("opens after consecutive failures reach the threshold", async () => { + const fail = async (): Promise => { + throw new Error("boom"); + }; + + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + expect(breaker.getState()).toBe("closed"); + + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + expect(breaker.getState()).toBe("open"); + expect(breaker.getStats().failureCount).toBe(3); + }); + + it("resets failureCount on success while closed so sparse failures do not trip", async () => { + const fail = async (): Promise => { + throw new Error("boom"); + }; + const succeed = async (): Promise => "ok"; + + // Two failures, then a success (still under threshold) + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + expect(breaker.getStats().failureCount).toBe(2); + + await expect(breaker.execute(succeed)).resolves.toBe("ok"); + expect(breaker.getState()).toBe("closed"); + expect(breaker.getStats().failureCount).toBe(0); + + // Two more failures after the success must not open the circuit + // (would have been 4 cumulative without the reset) + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + expect(breaker.getState()).toBe("closed"); + expect(breaker.getStats().failureCount).toBe(2); + + // A third consecutive failure does open it + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + expect(breaker.getState()).toBe("open"); + }); + + it("rejects requests while open", async () => { + const fail = async (): Promise => { + throw new Error("boom"); + }; + + for (let i = 0; i < 3; i++) { + await expect(breaker.execute(fail)).rejects.toThrow("boom"); + } + + await expect(breaker.execute(async () => "ok")).rejects.toThrow( + CircuitBreakerOpenError, + ); + }); + }); +});