Circuit Breakers for Downstream Failures

Why Cascading Failures Happen

The Circuit Breaker State Machine

A Minimal Breaker From Scratch

type State = 'CLOSED' | 'OPEN' | 'HALF_OPEN';

class MiniBreaker {
  private state: State = 'CLOSED';
  private failures = 0;
  private openedAt = 0;
  constructor(private threshold = 3, private resetMs = 1000) {}

  async call<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === 'OPEN') {
      if (Date.now() - this.openedAt >= this.resetMs) this.state = 'HALF_OPEN';
      else throw new Error('Circuit OPEN: failing fast');
    }
    try {
      const result = await fn();
      this.failures = 0;
      this.state = 'CLOSED';
      return result;
    } catch (e) {
      this.failures++;
      if (this.failures >= this.threshold) {
        this.state = 'OPEN';
        this.openedAt = Date.now();
      }
      throw e;
    }
  }
  get current() { return this.state; }
}

async function main() {
  const breaker = new MiniBreaker(2, 500);
  const flaky = () => Promise.reject(new Error('downstream down'));
  for (let i = 0; i < 4; i++) {
    try { await breaker.call(flaky); }
    catch (e) { console.log(`call ${i}: ${(e as Error).message} [state=${breaker.current}]`); }
  }
}
main();

Enter opossum

import CircuitBreaker from 'opossum';

const options: CircuitBreaker.Options = {
  timeout: 3000,                 // a call slower than 3s counts as a failure
  errorThresholdPercentage: 50,  // trip when >=50% of calls fail
  resetTimeout: 10000,           // stay OPEN for 10s, then try HALF_OPEN
  rollingCountTimeout: 10000,    // 10s statistical window
  rollingCountBuckets: 10,       // split the window into 10 buckets
};

// The action is the function we want to protect
async function fetchUser(id: string): Promise<{ id: string }> {
  // ... real HTTP call to a downstream user service ...
  return { id };
}

export const userBreaker = new CircuitBreaker(fetchUser, options);

Wrapping a Downstream Call in a NestJS Provider

import { Injectable } from '@nestjs/common';
import { HttpService } from '@nestjs/axios';
import { firstValueFrom } from 'rxjs';
import CircuitBreaker from 'opossum';

interface PricingDto { sku: string; cents: number; }

@Injectable()
export class PricingClient {
  private readonly breaker: CircuitBreaker<[string], PricingDto>;

  constructor(private readonly http: HttpService) {
    this.breaker = new CircuitBreaker(
      (sku: string) => this.requestPrice(sku),
      { timeout: 2000, errorThresholdPercentage: 50, resetTimeout: 15000 },
    );
  }

  private async requestPrice(sku: string): Promise<PricingDto> {
    const res = await firstValueFrom(
      this.http.get<PricingDto>(`https://pricing.internal/skus/${sku}`),
    );
    return res.data;
  }

  getPrice(sku: string): Promise<PricingDto> {
    return this.breaker.fire(sku);
  }
}

Fallbacks: Degrade Gracefully

import CircuitBreaker from 'opossum';

type Recommendation = { id: string };

function buildRecommendationBreaker(
  action: (userId: string) => Promise<Recommendation[]>,
) {
  const breaker = new CircuitBreaker(action, {
    timeout: 1500,
    errorThresholdPercentage: 40,
    resetTimeout: 20000,
  });

  // When OPEN or the action fails, return a safe empty list
  breaker.fallback((_userId: string, err?: Error) => {
    if (err) console.warn('recommendations degraded:', err.message);
    return [] as Recommendation[];
  });

  return breaker;
}

Listening to Breaker Events

import { Logger } from '@nestjs/common';
import CircuitBreaker from 'opossum';

export function attachBreakerTelemetry(
  breaker: CircuitBreaker,
  name: string,
  logger = new Logger('CircuitBreaker'),
) {
  breaker.on('open', () => logger.error(`[${name}] OPEN - failing fast`));
  breaker.on('halfOpen', () => logger.warn(`[${name}] HALF_OPEN - probing`));
  breaker.on('close', () => logger.log(`[${name}] CLOSED - recovered`));
  breaker.on('reject', () => logger.warn(`[${name}] call rejected (OPEN)`));
  breaker.on('timeout', () => logger.warn(`[${name}] call timed out`));
  breaker.on('fallback', () => logger.warn(`[${name}] fallback served`));
}

Timeout Is Part of the Breaker

import { Module } from '@nestjs/common';
import { HttpModule } from '@nestjs/axios';

@Module({
  imports: [
    HttpModule.register({
      timeout: 2500,        // axios aborts the socket at 2.5s
      maxRedirects: 0,
    }),
  ],
})
export class DownstreamModule {}

// Breaker timeout (e.g. 2000ms) should sit just BELOW the axios timeout
// so opossum records the failure while the socket is still being released.

One Breaker Per Dependency, Not Per App

import { Injectable } from '@nestjs/common';
import CircuitBreaker from 'opossum';

@Injectable()
export class BreakerRegistry {
  private readonly breakers = new Map<string, CircuitBreaker>();

  get<TArgs extends unknown[], TRet>(
    name: string,
    action: (...args: TArgs) => Promise<TRet>,
    options: CircuitBreaker.Options,
  ): CircuitBreaker<TArgs, TRet> {
    const existing = this.breakers.get(name);
    if (existing) return existing as CircuitBreaker<TArgs, TRet>;
    const breaker = new CircuitBreaker(action, options);
    this.breakers.set(name, breaker);
    return breaker;
  }
}

Exposing Breaker Health to Probes

import { Injectable } from '@nestjs/common';
import { HealthIndicator, HealthIndicatorResult } from '@nestjs/terminus';
import CircuitBreaker from 'opossum';

@Injectable()
export class BreakerHealthIndicator extends HealthIndicator {
  check(name: string, breaker: CircuitBreaker): HealthIndicatorResult {
    const isUp = !breaker.opened;
    return this.getStatus(name, isUp, {
      state: breaker.opened ? 'open' : breaker.halfOpen ? 'half_open' : 'closed',
      failures: breaker.stats.failures,
      timeouts: breaker.stats.timeouts,
      rejects: breaker.stats.rejects,
    });
  }
}

Tuning Thresholds Sanely

import CircuitBreaker from 'opossum';

const options: CircuitBreaker.Options = {
  timeout: 2000,
  errorThresholdPercentage: 50,
  resetTimeout: 15000,
  rollingCountTimeout: 10000,
  rollingCountBuckets: 10,
  volumeThreshold: 10, // need >=10 calls in the window before % can trip
  // errorFilter lets you NOT count expected errors (e.g. 404) as failures:
  errorFilter: (err: { statusCode?: number }) => err?.statusCode === 404,
};

export { options };

Quick Check: Choosing the Right Behavior

Recap