Redis, Polly & Observability In Depth

// Redis strengths in .NET microservices:
//   • Sub-millisecond reads for hot data (product catalog, config)
//   • Shared cache across multiple API instances (unlike IMemoryCache)
//   • TTL expiry — automatic eviction without manual cleanup
//   • Atomic counters — rate limiting, leaderboard scores
//   • Pub/Sub — lightweight event fan-out (not a message broker replacement)

// When NOT to use Redis as primary storage:
//   • Data you can't afford to lose (use SQL + cache-aside)
//   • Large blobs (> few MB per key — use blob storage)
//   • Complex queries (Redis is key/value + structures, not SQL)

# docker-compose.yml snippet
services:
  redis:
    image: redis:7-alpine
    ports: ["6379:6379"]
    command: redis-server --appendonly yes   # AOF persistence
    volumes: [redis-data:/data]

# appsettings.Development.json
{
  "Redis": {
    "ConnectionString": "localhost:6379,abortConnect=false,connectRetry=3"
  }
}

// Program.cs
builder.Services.AddStackExchangeRedisCache(options =>
{
    options.Configuration = builder.Configuration["Redis:ConnectionString"];
    options.InstanceName  = "ShopApi:";   // key prefix — avoids collisions
});

// Inject IDistributedCache
public sealed class ProductService(IDistributedCache cache, AppDbContext db)
{
    public async Task<ProductDto?> GetAsync(int id, CancellationToken ct)
    {
        var key = $"product:{id}";
        var bytes = await cache.GetAsync(key, ct);
        if (bytes is not null)
            return JsonSerializer.Deserialize<ProductDto>(bytes);

        var entity = await db.Products.FindAsync([id], ct);
        if (entity is null) return null;

        var dto = Map(entity);
        await cache.SetAsync(key,
            JsonSerializer.SerializeToUtf8Bytes(dto),
            new DistributedCacheEntryOptions
            {
                AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10)
            }, ct);
        return dto;
    }
}

var opts = new DistributedCacheEntryOptions();

// Absolute — expires at fixed time from now
opts.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1);

// Sliding — resets TTL on each read (good for session data)
opts.SlidingExpiration = TimeSpan.FromMinutes(20);

// Priority hint (Redis ignores this; IMemoryCache uses it under memory pressure)
opts.SetPriority(CacheItemPriority.High);

await cache.SetAsync("session:abc", data, opts, ct);
await cache.RemoveAsync("product:42", ct);   // explicit invalidation

// Register once — expensive to create, thread-safe, share everywhere
builder.Services.AddSingleton<IConnectionMultiplexer>(_ =>
    ConnectionMultiplexer.Connect(
        builder.Configuration["Redis:ConnectionString"]!,
        options => {
            options.AbortOnConnectFail = false;  // retry in background
            options.ConnectRetry       = 5;
        }));

// Usage
public sealed class LeaderboardService(IConnectionMultiplexer mux)
{
    private IDatabase Db => mux.GetDatabase();

    public async Task AddScoreAsync(string gameId, string player, double score)
        => await Db.SortedSetAddAsync($"lb:{gameId}", player, score);

    public async Task<SortedSetEntry[]> Top10Async(string gameId)
        => await Db.SortedSetRangeByRankWithScoresAsync($"lb:{gameId}", 0, 9,
               order: Order.Descending);
}

// Pub/Sub — fire-and-forget notifications (not durable)
var sub = mux.GetSubscriber();
await sub.PublishAsync(RedisChannel.Literal("orders:created"), orderId.ToString());

sub.Subscribe(RedisChannel.Literal("orders:created"), (_, msg) =>
    log.LogInformation("New order {OrderId}", msg));

// RedLock pattern — use RedLock.net for production distributed locks
// Simple SET NX with expiry (sketch only):
var acquired = await db.StringSetAsync(
    $"lock:invoice:{id}", Environment.MachineName,
    expiry: TimeSpan.FromSeconds(30),
    when: When.NotExists);

public async Task<CustomerDto?> GetCustomerAsync(int id, CancellationToken ct)
{
    var cacheKey = CacheKeys.Customer(id);
    var cached = await _cache.GetStringAsync(cacheKey, ct);
    if (cached is not null)
        return JsonSerializer.Deserialize<CustomerDto>(cached);

    var entity = await _repo.GetByIdAsync(id, ct);
    if (entity is null) return null;

    var dto = _mapper.Map<CustomerDto>(entity);
    await _cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(dto),
        new DistributedCacheEntryOptions {
            AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(15)
        }, ct);
    return dto;
}

// ✅ On update/delete — remove cache entry (simplest, safest)
public async Task UpdateCustomerAsync(int id, UpdateCustomerDto dto, CancellationToken ct)
{
    await _repo.UpdateAsync(id, dto, ct);
    await _cache.RemoveAsync(CacheKeys.Customer(id), ct);
    // Next read repopulates from DB
}

// ❌ Don't update cache AND DB in separate steps without transaction
//    — race conditions leave stale data

// Stampede protection — single-flight with SemaphoreSlim or FusionCache
private readonly SemaphoreSlim _gate = new(1, 1);
// Only one thread loads from DB when cache expires for hot keys

// Polly v8 — ResiliencePipeline
var pipeline = new ResiliencePipelineBuilder()
    .AddRetry(new RetryStrategyOptions
    {
        MaxRetryAttempts = 3,
        Delay            = TimeSpan.FromSeconds(1),
        BackoffType      = DelayBackoffType.Exponential,
        UseJitter        = true,   // avoid thundering herd
        ShouldHandle     = new PredicateBuilder()
            .Handle<HttpRequestException>()
            .Handle<TimeoutException>()
            .HandleResult<HttpResponseMessage>(r =>
                r.StatusCode is >= HttpStatusCode.InternalServerError
                             or HttpStatusCode.RequestTimeout)
    })
    .Build();

var response = await pipeline.ExecuteAsync(
    async ct => await http.GetAsync(url, ct), ct);

var pipeline = new ResiliencePipelineBuilder()
    .AddCircuitBreaker(new CircuitBreakerStrategyOptions
    {
        FailureRatio    = 0.5,                    // 50% failures trip breaker
        MinimumThroughput = 10,
        SamplingDuration  = TimeSpan.FromSeconds(30),
        BreakDuration     = TimeSpan.FromSeconds(60),
        OnOpened  = args => log.LogWarning("Circuit OPEN for {Duration}s", args.BreakDuration),
        OnClosed  = _    => log.LogInformation("Circuit CLOSED — healthy again"),
        OnHalfOpened = _ => log.LogInformation("Circuit HALF-OPEN — probing")
    })
    .AddRetry(new RetryStrategyOptions { MaxRetryAttempts = 2 })
    .Build();

// When circuit is open → BrokenCircuitException immediately
// No wasted threads hammering a dead Redis/SQL/API

// NuGet: AspNetCore.HealthChecks.Redis
builder.Services.AddHealthChecks()
    .AddRedis(builder.Configuration["Redis:ConnectionString"]!,
        name: "redis", tags: ["ready", "cache"])
    .AddNpgSql(builder.Configuration.GetConnectionString("Default")!,
        name: "postgres", tags: ["ready", "db"])
    .AddUrlGroup(new Uri("https://payments-api/health"),
        name: "payments-api", tags: ["ready", "external"]);

// Liveness — process is up (no dependency checks)
app.MapHealthChecks("/health/live", new HealthCheckOptions {
    Predicate = _ => false
});

// Readiness — can serve traffic (all "ready" tags must pass)
app.MapHealthChecks("/health/ready", new HealthCheckOptions {
    Predicate = check => check.Tags.Contains("ready"),
    ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse
});

public sealed class RedisHealthCheck(IConnectionMultiplexer mux) : IHealthCheck
{
    public async Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext ctx, CancellationToken ct)
    {
        try
        {
            var db = mux.GetDatabase();
            var pong = await db.PingAsync();
            return pong.TotalMilliseconds < 100
                ? HealthCheckResult.Healthy($"Ping {pong.TotalMilliseconds:F1}ms")
                : HealthCheckResult.Degraded($"Slow ping {pong.TotalMilliseconds:F1}ms");
        }
        catch (Exception ex)
        {
            return HealthCheckResult.Unhealthy("Redis unreachable", ex);
        }
    }
}

// Program.cs — bootstrap logger catches startup failures
Log.Logger = new LoggerConfiguration()
    .WriteTo.Console()
    .CreateBootstrapLogger();

builder.Host.UseSerilog((ctx, services, config) => config
    .ReadFrom.Configuration(ctx.Configuration)
    .ReadFrom.Services(services)
    .Enrich.FromLogContext()
    .Enrich.WithProperty("Application", "ShopApi")
    .Enrich.WithMachineName()
    .WriteTo.Console(new JsonFormatter())
    .WriteTo.Seq(ctx.Configuration["Seq:Url"]!));

// Structured — properties are queryable fields, NOT string concat
log.LogInformation("Order {OrderId} placed by {CustomerId} in {ElapsedMs}ms",
    order.Id, order.CustomerId, sw.ElapsedMilliseconds);

// ❌ log.LogInformation($"Order {order.Id} placed");  // one blob string

app.Use(async (ctx, next) =>
{
    var correlationId = ctx.Request.Headers["X-Correlation-ID"].FirstOrDefault()
        ?? Activity.Current?.Id ?? Guid.NewGuid().ToString("N");
    ctx.Response.Headers["X-Correlation-ID"] = correlationId;

    using (LogContext.PushProperty("CorrelationId", correlationId))
    using (LogContext.PushProperty("TraceId", Activity.Current?.TraceId.ToString()))
    {
        await next(ctx);
    }
});

// Every log in the request pipeline carries CorrelationId
// Trace a single checkout across API → Redis → Payment service

builder.Services.AddOpenTelemetry()
    .ConfigureResource(r => r.AddService("ShopApi"))
    .WithTracing(t => t
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRedisInstrumentation()          // StackExchange.Redis spans
        .AddEntityFrameworkCoreInstrumentation()
        .AddOtlpExporter())                 // or .AddAzureMonitorTraceExporter()
    .WithMetrics(m => m
        .AddAspNetCoreInstrumentation()
        .AddRuntimeInstrumentation()
        .AddMeter("ShopApi.Cache")          // custom business metrics
        .AddOtlpExporter());

// Custom span around cache operation
using var activity = ActivitySource.StartActivity("Cache.Get");
activity?.SetTag("cache.key", key);
activity?.SetTag("cache.hit", hit);

// NuGet: Azure.Monitor.OpenTelemetry.AspNetCore
builder.Services.AddOpenTelemetry()
    .UseAzureMonitor(options =>
        options.ConnectionString = builder.Configuration["ApplicationInsights:ConnectionString"]);

// Automatic: HTTP requests, dependencies (SQL, Redis, outbound HTTP)
// Custom events:
var telemetry = services.GetRequiredService<TelemetryClient>();
telemetry.TrackMetric("CacheHitRate", hitRate);
telemetry.TrackEvent("CircuitBreakerOpened", new Dictionary<string, string> {
    ["Service"] = "payments-api"
});

// NuGet: Microsoft.Extensions.Http.Resilience
builder.Services.AddHttpClient("payments-api", client =>
{
    client.BaseAddress = new Uri(builder.Configuration["Payments:BaseUrl"]!);
    client.Timeout     = TimeSpan.FromSeconds(30);
})
.AddStandardResilienceHandler(options =>
{
    options.Retry.MaxRetryAttempts = 3;
    options.CircuitBreaker.SamplingDuration = TimeSpan.FromSeconds(30);
    options.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(60);
});

// Typed client
public sealed class PaymentClient(HttpClient http)
{
    public Task<PaymentResult> ChargeAsync(ChargeRequest req, CancellationToken ct)
        => http.PostAsJsonAsync("/charge", req, ct)
               .ContinueWith(t => t.Result.EnsureSuccessStatusCode(), ct)
               .Unwrap();
}

public sealed class OutboundHeadersHandler(
    IHttpContextAccessor httpContext) : DelegatingHandler
{
    protected override Task<HttpResponseMessage> SendAsync(
        HttpRequestMessage request, CancellationToken ct)
    {
        var ctx = httpContext.HttpContext;
        if (ctx is not null)
        {
            if (ctx.Request.Headers.TryGetValue("X-Correlation-ID", out var cid))
                request.Headers.TryAddWithoutValidation("X-Correlation-ID", cid.ToString());

            var token = ctx.Request.Headers.Authorization.ToString();
            if (!string.IsNullOrEmpty(token))
                request.Headers.TryAddWithoutValidation("Authorization", token);
        }
        return base.SendAsync(request, ct);
    }
}

builder.Services.AddTransient<OutboundHeadersHandler>();
builder.Services.AddHttpClient<PaymentClient>()
    .AddHttpMessageHandler<OutboundHeadersHandler>()
    .AddStandardResilienceHandler();

// Production resilience stack (typical):
// 1. IHttpClientFactory + StandardResilienceHandler
// 2. Redis cache-aside with TTL + invalidation on write
// 3. Health checks on /health/ready → K8s/Azure probes
// 4. Serilog → Seq/App Insights with CorrelationId
// 5. OpenTelemetry traces for slow dependency diagnosis
// 6. Alerts on circuit breaker open + cache hit rate drop

// Simple key/value cache across instances?
//   → IDistributedCache + Redis

// Sorted sets, pub/sub, atomic counters?
//   → StackExchange.Redis directly

// Flaky HTTP/RPC to another service?
//   → IHttpClientFactory + Polly (retry + circuit breaker)

// Need to know if app can serve traffic?
//   → Health checks (/health/live vs /health/ready)

// Debug slow requests across services?
//   → OpenTelemetry traces + CorrelationId in Serilog

// Cross-service messaging with guaranteed delivery?
//   → MassTransit / Service Bus (NOT Redis pub/sub)