10 - Sampling Strategies - Reducing Telemetry Volume and Costs

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.sampling import (
    TraceIdRatioBased,
    ParentBasedTraceIdRatio,
    ALWAYS_ON,
    ALWAYS_OFF,
)

# --- 1. Probability Sampler (25% of traces) ---
# Each trace has a 25% probability of being sampled
sampler_25 = TraceIdRatioBased(0.25)

# --- 2. ParentBased Sampler (respects parent decision) ---
# If parent is sampled, child is sampled
# If no parent, uses the specified ratio
sampler_parent = ParentBasedTraceIdRatio(0.25)

# --- 3. Configuration in TracerProvider ---
provider = TracerProvider(
    sampler=ParentBasedTraceIdRatio(0.25)
)
trace.set_tracer_provider(provider)

# --- 4. Configuration via environment variables ---
# OTEL_TRACES_SAMPLER=parentbased_traceidratio
# OTEL_TRACES_SAMPLER_ARG=0.25
      

# tail_sampling configuration in Gateway Collector
processors:
  tail_sampling:
    # Maximum time to wait for all spans of a trace
    decision_wait: 30s
    # Maximum number of traces awaiting decision
    num_traces: 100000
    # Expected new traces per second
    expected_new_traces_per_sec: 1000

    policies:
      # Policy 1: keep ALL traces with errors
      - name: errors-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR

      # Policy 2: keep slow traces (latency > 2s)
      - name: latency-policy
        type: latency
        latency:
          threshold_ms: 2000

      # Policy 3: keep traces with specific attributes
      - name: vip-customers
        type: string_attribute
        string_attribute:
          key: customer.tier
          values:
            - premium
            - enterprise

      # Policy 4: probabilistic sampling for the rest
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

      # Policy 5: global rate limiting
      - name: rate-limiting
        type: rate_limiting
        rate_limiting:
          spans_per_second: 500

      # Policy 6: composite policy (AND/OR)
      - name: composite-policy
        type: composite
        composite:
          max_total_spans_per_second: 1000
          policy_order: [errors-policy, latency-policy, probabilistic-policy]
          rate_allocation:
            - policy: errors-policy
              percent: 50
            - policy: latency-policy
              percent: 30
            - policy: probabilistic-policy
              percent: 20
      

# Multi-level optimization strategy

# 1. Filter noise in the Collector
processors:
  filter:
    traces:
      span:
        # Eliminate health checks and readiness probes
        - 'attributes["http.route"] == "/health"'
        - 'attributes["http.route"] == "/ready"'
        - 'attributes["http.route"] == "/metrics"'
        # Eliminate static asset requests
        - 'attributes["http.route"] =~ "/static/.*"'

  # 2. Remove high-cardinality attributes
  attributes:
    actions:
      # Remove sensitive headers
      - key: http.request.header.authorization
        action: delete
      - key: http.request.header.cookie
        action: delete
      # Hash long SQL queries
      - key: db.query.text
        action: hash

  # 3. Intelligent tail sampling
  tail_sampling:
    policies:
      - name: keep-errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: keep-slow
        type: latency
        latency:
          threshold_ms: 1000
      - name: sample-rest
        type: probabilistic
        probabilistic:
          sampling_percentage: 5

  # 4. Batch for efficiency
  batch:
    send_batch_size: 2048
    timeout: 10s