# tracing.py
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor

def configure_tracing(service_name: str, otlp_endpoint: str = "http://jaeger-collector:4317"):
    resource = Resource(attributes={
        SERVICE_NAME: service_name,
        "service.version": "1.0.0",
        "deployment.environment": "production",
    })

    provider = TracerProvider(resource=resource)

    otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
    provider.add_span_processor(BatchSpanProcessor(otlp_exporter))

    trace.set_tracer_provider(provider)

    # Auto-instrument frameworks
    FastAPIInstrumentor().instrument()
    HTTPXClientInstrumentor().instrument()

    return trace.get_tracer(service_name)

# main.py
from fastapi import FastAPI
from tracing import configure_tracing
import httpx

app = FastAPI()
tracer = configure_tracing("order-service")

@app.get("/orders/{order_id}")
async def get_order(order_id: str):
    # Manual span for custom operations
    with tracer.start_as_current_span("fetch-order-details") as span:
        span.set_attribute("order.id", order_id)

        # This HTTP call will be auto-instrumented
        async with httpx.AsyncClient() as client:
            user = await client.get(f"http://user-service/users/{order_id}")

        span.set_attribute("user.id", user.json()["id"])
        return {"order_id": order_id, "user": user.json()}

# Python: add custom spans for business logic
from opentelemetry import trace

tracer = trace.get_tracer(__name__)

def process_payment(payment_id: str, amount: float):
    with tracer.start_as_current_span("process-payment") as span:
        span.set_attribute("payment.id", payment_id)
        span.set_attribute("payment.amount", amount)
        span.set_attribute("payment.currency", "USD")

        try:
            result = charge_card(payment_id, amount)
            span.set_attribute("payment.status", "success")
            span.set_attribute("payment.transaction_id", result.transaction_id)
            return result
        except PaymentDeclinedException as e:
            span.set_attribute("payment.status", "declined")
            span.record_exception(e)
            span.set_status(trace.Status(trace.StatusCode.ERROR, str(e)))
            raise

from opentelemetry.sdk.trace.sampling import (
    TraceIdRatioBased,
    ParentBased,
    ALWAYS_ON,
    ALWAYS_OFF,
)

# Sample 10% of traces in production
sampler = ParentBased(
    root=TraceIdRatioBased(0.1),
    remote_parent_sampled=ALWAYS_ON,    # Always sample if parent was sampled
    remote_parent_not_sampled=ALWAYS_OFF,
)

provider = TracerProvider(resource=resource, sampler=sampler)

# Producer: inject trace context into message headers
from opentelemetry import trace, propagate
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator

def publish_order_event(channel, order_id: str, payload: dict):
    tracer = trace.get_tracer(__name__)

    with tracer.start_as_current_span("publish-order-event") as span:
        span.set_attribute("messaging.system", "rabbitmq")
        span.set_attribute("messaging.destination", "orders")
        span.set_attribute("order.id", order_id)

        headers = {}
        propagate.inject(headers)  # Injects traceparent + tracestate

        channel.basic_publish(
            exchange="orders",
            routing_key="order.created",
            body=json.dumps(payload),
            properties=pika.BasicProperties(headers=headers)
        )

# Consumer: extract trace context from message headers
def process_message(channel, method, properties, body):
    tracer = trace.get_tracer(__name__)
    ctx = propagate.extract(properties.headers or {})

    with tracer.start_as_current_span(
        "process-order-event",
        context=ctx,
        kind=trace.SpanKind.CONSUMER
    ) as span:
        span.set_attribute("messaging.system", "rabbitmq")
        order = json.loads(body)
        handle_order(order)

#!/usr/bin/env python3
# scripts/jaeger-alert.py — run every 5 minutes via cron
import requests, json, os, sys
from datetime import datetime, timedelta

JAEGER_URL = os.getenv("JAEGER_URL", "http://localhost:16686")
SLACK_WEBHOOK = os.getenv("SLACK_WEBHOOK")
ERROR_THRESHOLD = int(os.getenv("ERROR_THRESHOLD", "10"))
SERVICES = ["order-service", "payment-service", "user-service"]

now = datetime.utcnow()
five_min_ago = now - timedelta(minutes=5)
start_us = int(five_min_ago.timestamp() * 1_000_000)
end_us   = int(now.timestamp() * 1_000_000)

alerts = []
for service in SERVICES:
    resp = requests.get(
        f"{JAEGER_URL}/api/traces",
        params={
            "service": service,
            "tags": '{"error":"true"}',
            "start": start_us,
            "end": end_us,
            "limit": 100,
        },
        timeout=10,
    )
    traces = resp.json().get("data", [])
    if len(traces) >= ERROR_THRESHOLD:
        alerts.append(f"*{service}*: {len(traces)} error traces in the last 5 minutes")

if alerts and SLACK_WEBHOOK:
    payload = {"text": "Jaeger trace alert:\n" + "\n".join(alerts)}
    requests.post(SLACK_WEBHOOK, json=payload, timeout=5)
    sys.exit(1)

print("No anomalies detected.")

# logging_config.py
import logging
from opentelemetry._logs import set_logger_provider
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter

def configure_logging(service_name: str):
    logger_provider = LoggerProvider()
    otlp_exporter = OTLPLogExporter(endpoint="http://jaeger-collector:4317", insecure=True)
    logger_provider.add_log_record_processor(BatchLogRecordProcessor(otlp_exporter))
    set_logger_provider(logger_provider)

    handler = LoggingHandler(level=logging.DEBUG, logger_provider=logger_provider)

    # Add trace context fields to every log record
    logging.basicConfig(handlers=[handler], level=logging.INFO)
    return logging.getLogger(service_name)