12 - Case Study - End-to-End Observability Implementation (Microservices)

# Phase 1: Deploy OTel Collector as DaemonSet
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector-agent
  namespace: observability
spec:
  selector:
    matchLabels:
      app: otel-agent
  template:
    spec:
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.96.0
          args: ["--config=/etc/otel/config.yaml"]
          env:
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
          ports:
            - containerPort: 4317
              hostPort: 4317
---
# Auto-instrumentation for Java services
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: shopflow-instrumentation
  namespace: shopflow
spec:
  exporter:
    endpoint: http://otel-collector-agent.observability:4317
  propagators:
    - tracecontext
    - baggage
  sampler:
    type: parentbased_traceidratio
    argument: "1.0"  # 100% in initial phase
  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:latest
  python:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:latest
  nodejs:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:latest
---
# Annotate deployments for auto-instrumentation
# Order Service (Java)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  namespace: shopflow
spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-java: "shopflow-instrumentation"
      

// Order Service: manual instrumentation of checkout flow
@Service
public class CheckoutService {

    private final Tracer tracer;
    private final LongCounter ordersCreated;
    private final DoubleHistogram orderValue;

    public CheckoutService(OpenTelemetry otel) {
        this.tracer = otel.getTracer("order-service");
        Meter meter = otel.getMeter("order-service");
        this.ordersCreated = meter.counterBuilder("shopflow.orders.created")
            .setDescription("Orders created").build();
        this.orderValue = meter.histogramBuilder("shopflow.orders.value")
            .setDescription("Order value in EUR").setUnit("EUR").build();
    }

    public Order processCheckout(CheckoutRequest req) {
        Span span = tracer.spanBuilder("checkout.process")
            .setAttribute("customer.id", req.getCustomerId())
            .setAttribute("customer.tier", req.getCustomerTier())
            .setAttribute("cart.items_count", req.getItems().size())
            .setAttribute("cart.total", req.getTotal())
            .startSpan();

        try (Scope scope = span.makeCurrent()) {
            // Validation
            Span valSpan = tracer.spanBuilder("checkout.validate").startSpan();
            try (Scope s = valSpan.makeCurrent()) {
                validateCheckout(req);
                valSpan.setStatus(StatusCode.OK);
            } finally { valSpan.end(); }

            // Reserve inventory
            Span invSpan = tracer.spanBuilder("checkout.reserve-inventory")
                .setAttribute("inventory.items", req.getItems().size())
                .startSpan();
            try (Scope s = invSpan.makeCurrent()) {
                reserveInventory(req.getItems());
            } finally { invSpan.end(); }

            // Payment
            Span paySpan = tracer.spanBuilder("checkout.payment")
                .setAttribute("payment.method", req.getPaymentMethod())
                .setAttribute("payment.amount", req.getTotal())
                .startSpan();
            try (Scope s = paySpan.makeCurrent()) {
                processPayment(req);
            } finally { paySpan.end(); }

            // Create order
            Order order = createOrder(req);
            span.setAttribute("order.id", order.getId());
            span.setAttribute("order.status", "created");

            // Business metrics
            ordersCreated.add(1, Attributes.of(
                AttributeKey.stringKey("customer.tier"), req.getCustomerTier(),
                AttributeKey.stringKey("payment.method"), req.getPaymentMethod()
            ));
            orderValue.record(req.getTotal(), Attributes.of(
                AttributeKey.stringKey("customer.tier"), req.getCustomerTier()
            ));

            span.setStatus(StatusCode.OK);
            return order;
        } catch (Exception e) {
            span.recordException(e);
            span.setStatus(StatusCode.ERROR, e.getMessage());
            throw e;
        } finally {
            span.end();
        }
    }
}
      

# SLO-based alerts for ShopFlow
groups:
  - name: shopflow-slo-alerts
    rules:
      # SLO: 99.5% checkout requests successful
      - alert: CheckoutSLOBreach
        expr: |
          1 - (
            sum(rate(http_server_request_duration_seconds_count{
              service="order-service",
              http_route="/api/checkout",
              http_status_code=~"2.."
            }[1h]))
            /
            sum(rate(http_server_request_duration_seconds_count{
              service="order-service",
              http_route="/api/checkout"
            }[1h]))
          ) > 0.005
        for: 5m
        labels:
          severity: critical
          slo: checkout-success-rate
        annotations:
          summary: "Checkout success rate below 99.5% SLO"

      # SLO: P99 checkout latency under 3 seconds
      - alert: CheckoutLatencySLO
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_server_request_duration_seconds_bucket{
              service="order-service",
              http_route="/api/checkout"
            }[5m])) by (le)
          ) > 3
        for: 5m
        labels:
          severity: warning
          slo: checkout-latency
        annotations:
          summary: "Checkout P99 latency above 3s SLO"

      # SLO: payment success above 98%
      - alert: PaymentSuccessRateSLO
        expr: |
          sum(rate(shopflow_payments_total{status="success"}[1h]))
          /
          sum(rate(shopflow_payments_total[1h])) < 0.98
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Payment success rate below 98%"