Resilience Patterns

Resilience patterns help systems handle failures gracefully, maintaining functionality even when components fail.

Circuit Breaker Pattern

// Polly Circuit Breaker
public class ResilientHttpClient
{
    private readonly AsyncCircuitBreakerPolicy<HttpResponseMessage> _circuitBreaker;
    
    public ResilientHttpClient(HttpClient httpClient)
    {
        _circuitBreaker = Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => r.StatusCode >= HttpStatusCode.InternalServerError)
            .CircuitBreakerAsync(
                handledEventsAllowedBeforeBreaking: 5,
                durationOfBreak: TimeSpan.FromSeconds(30),
                onBreak: (result, duration) => _logger.LogWarning(
                    "Circuit opened for {Duration}s", duration.TotalSeconds),
                onReset: () => _logger.LogInformation("Circuit reset"));
    }
}

Retry with Exponential Backoff

public static IAsyncPolicy<HttpResponseMessage> GetRetryPolicy()
{
    return Policy<HttpResponseMessage>
        .Handle<HttpRequestException>()
        .OrResult(r => r.StatusCode >= HttpStatusCode.InternalServerError)
        .WaitAndRetryAsync(
            retryCount: 3,
            sleepDurationProvider: (retryAttempt, response, context) =>
            {
                // Exponential backoff with jitter
                var exponentialBackoff = TimeSpan.FromSeconds(Math.Pow(2, retryAttempt));
                var jitter = TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000));
                return exponentialBackoff + jitter;
            });
}

Bulkhead Pattern

// Separate thread pools for different operations
services.AddResiliencePipeline("critical-operations", builder =>
{
    builder.AddConcurrencyLimiter(new ConcurrencyLimiterOptions
    {
        PermitLimit = 10,
        QueueLimit = 5
    });
});

services.AddResiliencePipeline("background-operations", builder =>
{
    builder.AddConcurrencyLimiter(new ConcurrencyLimiterOptions
    {
        PermitLimit = 50,
        QueueLimit = 100
    });
});

Fallback Pattern

public async Task<Product> GetProductAsync(Guid productId)
{
    var fallbackPolicy = Policy<Product>
        .Handle<Exception>()
        .FallbackAsync(async ct =>
        {
            // Try cache as fallback
            var cached = await _cache.GetAsync<Product>($"product:{productId}");
            if (cached != null) return cached;
            
            // Return degraded response
            return new Product { Id = productId, Name = "Product Unavailable", IsAvailable = false };
        });
    
    return await fallbackPolicy.ExecuteAsync(() => _apiClient.GetProductAsync(productId));
}

Health Checks

builder.Services.AddHealthChecks()
    .AddCheck<DatabaseHealthCheck>("database")
    .AddCheck<ExternalApiHealthCheck>("external-api");

app.MapHealthChecks("/health/live", new HealthCheckOptions
{
    Predicate = _ => false // Just check if app is running
});

app.MapHealthChecks("/health/ready", new HealthCheckOptions
{
    Predicate = check => check.Tags.Contains("ready")
});

Graceful Degradation

public async Task<SearchResults> SearchAsync(SearchQuery query)
{
    var results = new SearchResults();
    
    try
    {
        results.Products = await _elasticSearch.SearchAsync(query);
    }
    catch (Exception ex)
    {
        _logger.LogWarning(ex, "Elasticsearch unavailable, using database");
        results.Products = await _database.SearchProductsAsync(query);
        results.IsDegraded = true;
    }
    
    // Optional enrichment - don't fail if unavailable
    try
    {
        results.Recommendations = await _recommendationService.GetRecommendationsAsync(query.UserId);
    }
    catch
    {
        results.Recommendations = new List<Product>();
    }
    
    return results;
}

Key Takeaways

1. Circuit Breaker: Prevent cascade failures by stopping calls to failing services
2. Retry with Backoff: Handle transient failures with exponential backoff and jitter
3. Bulkhead: Isolate failures to prevent resource exhaustion
4. Timeout: Prevent indefinite waits with appropriate timeouts
5. Fallback: Provide degraded functionality when services fail
6. Health Checks: Enable orchestrators to manage service lifecycle

---