Circuit Breaker Pattern: Stopping Cascading Failures in Distributed Systems

Time 0:   100 users/sec hitting /api/recommendations
          RecommendationService is healthy - avg 50ms response

Time 1m:  RecommendationService DB becomes slow (disk I/O saturation)
          RecommendationService now takes 30 seconds to respond

Time 2m:  API server creates a new thread per pending request
          100 users/sec x 30 seconds lag = 3,000 concurrent threads waiting

Time 3m:  API server reaches max thread pool (500 threads)
          API server now queues ALL requests - login fails, checkout fails

Time 4m:  Users cannot log in - the entire site is down
          Root cause: a slow RECOMMENDATION widget

# Resilience4j configuration (application.yml):
resilience4j:
  circuitbreaker:
    instances:
      recommendationService:
        registerHealthIndicator: true
        
        # Sliding window: count-based (last N calls) or time-based (last N seconds):
        slidingWindowType: COUNT_BASED
        slidingWindowSize: 20         # Evaluate based on last 20 calls
        
        # Trip when 50%+ of calls fail:
        failureRateThreshold: 50
        
        # Also trip when 60%+ of calls take > 3 seconds:
        slowCallRateThreshold: 60
        slowCallDurationThreshold: 3000ms
        
        # Minimum calls before evaluating failure rate:
        minimumNumberOfCalls: 5
        
        # How long to stay OPEN before testing:
        waitDurationInOpenState: 30s
        
        # How many trial calls in HALF-OPEN state:
        permittedNumberOfCallsInHalfOpenState: 5
        
        # Which exceptions count as failures:
        recordExceptions:
          - java.io.IOException
          - java.util.concurrent.TimeoutException
        
        # Which exceptions do NOT trip the circuit:
        ignoreExceptions:
          - com.myapp.exceptions.BusinessRuleException  # Not a service fault

@Service
public class RecommendationClient {
    private final WebClient webClient;
    private final CircuitBreakerRegistry cbRegistry;
    
    @CircuitBreaker(name = "recommendationService", fallbackMethod = "getStaticRecommendations")
    public List<Product> getPersonalizedRecommendations(String userId) {
        return webClient.get()
            .uri("/recommendations/{userId}", userId)
            .retrieve()
            .bodyToFlux(Product.class)
            .collectList()
            .timeout(Duration.ofSeconds(3))  // Combine with timeout!
            .block();
    }
    
    // Fallback: called when circuit is OPEN or call fails
    public List<Product> getStaticRecommendations(String userId, CallNotPermittedException ex) {
        log.warn("Circuit OPEN for recommendations. User: {}", userId);
        return productRepository.findTop10Bestsellers(); // Degraded fallback
    }
    
    public List<Product> getStaticRecommendations(String userId, TimeoutException ex) {
        log.warn("Recommendation timeout. Returning cached. User: {}", userId);
        return redisCache.getOrDefault("top10", List.of()); // Cached fallback
    }
}

@Bulkhead(
    name = "recommendationService",
    type = Bulkhead.Type.THREADPOOL,
    fallbackMethod = "getStaticRecommendations"
)
public CompletableFuture<List<Product>> getRecommendationsAsync(String userId) {
    return CompletableFuture.supplyAsync(
        () -> webClient.get().uri("/recommendations/{userId}", userId)...
    );
}

// Configuration: max 10 threads for recommendation service
# Only 10 threads can wait for recommendation - chat, login, checkout are unaffected
resilience4j.thread-pool-bulkhead.instances.recommendationService.maxThreadPoolSize: 10
resilience4j.thread-pool-bulkhead.instances.recommendationService.coreThreadPoolSize: 5

// ❌ WRONG: Retry wraps Circuit Breaker
// If CB is OPEN, retry tries 3 times x 100ms = 3 fallback calls (not wrong, but wasteful)
@Retry(name = "recSvc")
@CircuitBreaker(name = "recSvc")  // WRONG ORDER
public Product getRecommendation(String userId) { ... }

// ✅ CORRECT: Circuit Breaker wraps Retry
// Retry tries 3 times -> CB counts failures -> trips after threshold
@CircuitBreaker(name = "recSvc")  
@Retry(name = "recSvc")           // CORRECT ORDER - CB sees aggregated results
public Product getRecommendation(String userId) { ... }

// Even better: Use separate names + Retry for transient network errors only
// CB for detecting genuinely failing services

# Spring Boot Actuator exposes CB health:
management:
  endpoints.web.exposure.include: health,metrics,circuitbreakerevents
  health.circuitbreakers.enabled: true

# Prometheus metrics exposed:
# resilience4j_circuitbreaker_state{name="recommendationService"} 1.0
# 0 = CLOSED, 1 = OPEN, 2 = HALF_OPEN
# resilience4j_circuitbreaker_failure_rate{name="recommendationService"} 0.65