name: agent-observability description: Instrument AI agents with tracing, token metrics, latency, and cost visibility. Use for reliability and debugging. category: devops risk: critical source: https://github.com/BagelHole/DevOps-Security-Agent-Skills source_repo: BagelHole/DevOps-Security-Agent-Skills source_type: community date_added: '2026-09-20' license: MIT license_source: https://github.com/BagelHole/DevOps-Security-Agent-Skills/blob/main/LICENSE compatibility: Requires the relevant platform CLIs (kubectl, helm, terraform, git, CI runners) and authorized access to the target environment. Docs-only; helper scripts and templates not bundled. metadata: author: devops-skills version: '2.0'

Agent Observability

Monitor AI agent behavior with logs, traces, metrics, and cost telemetry. This skill covers the full observability stack for LLM-powered applications: from raw Prometheus counters to Grafana dashboards, OpenTelemetry tracing, structured logging, cost tracking, SLO definition, and PII redaction.

Core Metrics

Define these metrics at the application layer. All examples use the Prometheus client library naming conventions.

Latency

from prometheus_client import Histogram

# Total end-to-end latency for a full agent turn (user prompt -> final response)
AGENT_LATENCY = Histogram(
    "agent_request_duration_seconds",
    "End-to-end latency of an agent request",
    labelnames=["agent_name", "model", "status"],
    buckets=(0.25, 0.5, 1, 2, 5, 10, 30, 60, 120),
)

# Latency of a single LLM API call (one completion request)
LLM_CALL_LATENCY = Histogram(
    "llm_call_duration_seconds",
    "Latency of an individual LLM API call",
    labelnames=["model", "provider", "stream"],
    buckets=(0.1, 0.25, 0.5, 1, 2, 5, 10, 30),
)

# Latency of tool/function calls executed by the agent
TOOL_CALL_LATENCY = Histogram(
    "agent_tool_call_duration_seconds",
    "Latency of a tool call executed by the agent",
    labelnames=["tool_name", "agent_name", "status"],
    buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10),
)

# llm_tracing.py import time from opentelemetry import trace from opentelemetry.trace import StatusCode tracer = trace.get_tracer("agent.llm") def traced_llm_call(client, messages, model="gpt-4o", **kwargs): """Wrap an LLM completion call with a full OpenTelemetry span.""" with tracer.start_as_current_span("llm.chat_completion") as span: span.set_attribute("llm.model", model) span.set_attribute("llm.provider", "openai") span.set_attribute("llm.message_count", len(messages)) span.set_attribute("llm.temperature", kwargs.get("temperature", 1.0)) span.set_attribute("llm.max_tokens", kwargs.get("max_tokens", 0)) start = time.perf_counter() try: response = client.chat.completions.create( model=model, messages=messages, **kwargs ) elapsed = time.perf_counter() - start usage = response.usage span.set_attribute("llm.prompt_tokens", usage.prompt_tokens) span.set_attribute("llm.completion_tokens", usage.completion_tokens) span.set_attribute("llm.total_tokens", usage.total_tokens) span.set_attribute("llm.duration_seconds", elapsed) span.set_attribute("llm.finish_reason", response.choices[0].finish_reason) span.set_status(StatusCode.OK) # Update Prometheus counters PROMPT_TOKENS.labels(model=model, agent_name="default").inc(usage.prompt_tokens) COMPLETION_TOKENS.labels(model=model, agent_name="default").inc(usage.completion_tokens) LLM_CALL_LATENCY.labels(model=model, provider="openai", stream="false").observe(elapsed) return response except Exception as exc: elapsed = time.perf_counter() - start span.set_status(StatusCode.ERROR, str(exc)) span.record_exception(exc) LLM_ERRORS.labels(model=model, provider="openai", error_type=type(exc).__name__).inc() raise

# tool_tracing.py import functools from opentelemetry import trace from opentelemetry.trace import StatusCode tracer = trace.get_tracer("agent.tools") def traced_tool(tool_name: str): """Decorator that wraps a tool function with an OTel span and Prometheus metrics.""" def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): with tracer.start_as_current_span(f"tool.{tool_name}") as span: span.set_attribute("tool.name", tool_name) span.set_attribute("tool.args_count", len(args) + len(kwargs)) import time start = time.perf_counter() try: result = func(*args, **kwargs) elapsed = time.perf_counter() - start span.set_attribute("tool.duration_seconds", elapsed) span.set_status(StatusCode.OK) TOOL_CALLS_TOTAL.labels( tool_name=tool_name, agent_name="default", status="success" ).inc() TOOL_CALL_LATENCY.labels( tool_name=tool_name, agent_name="default", status="success" ).observe(elapsed) return result except Exception as exc: elapsed = time.perf_counter() - start span.set_status(StatusCode.ERROR, str(exc)) span.record_exception(exc) TOOL_CALLS_TOTAL.labels( tool_name=tool_name, agent_name="default", status="error" ).inc() TOOL_CALL_LATENCY.labels( tool_name=tool_name, agent_name="default", status="error" ).observe(elapsed) raise return wrapper return decorator # Usage @traced_tool("web_search") def web_search(query: str) -> str: # ... tool implementation ... pass @traced_tool("sql_query") def sql_query(statement: str) -> list: # ... tool implementation ... pass

agent-observability

Install

Related skills