Monitoring the verification endpoint¶

from fastapi import FastAPI, Request, Response
from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST
import time

VERIFICATIONS = Counter(
    "agent_manifest_verifications_total",
    "Total verification requests",
    ["result"],
)
VERIFICATION_LATENCY = Histogram(
    "agent_manifest_verification_duration_seconds",
    "End-to-end verification latency",
    buckets=[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0],
)
REVOCATION_LATENCY = Histogram(
    "agent_manifest_revocation_check_duration_seconds",
    "CRL lookup latency",
    buckets=[0.001, 0.005, 0.01, 0.05, 0.1, 0.5],
)
ACTIVE_MANIFESTS = Gauge(
    "agent_manifest_manifests_active",
    "Active manifests by attestation level",
    ["attestation_level"],
)

app = FastAPI()

@app.middleware("http")
async def record_verification_metrics(request: Request, call_next):
    if request.url.path == "/verify":
        start = time.perf_counter()
        response = await call_next(request)
        duration = time.perf_counter() - start
        VERIFICATION_LATENCY.observe(duration)
        return response
    return await call_next(request)

@app.get("/metrics")
def metrics():
    return Response(generate_latest(), media_type=CONTENT_TYPE_LATEST)

from opentelemetry import metrics
from opentelemetry.sdk.metrics import MeterProvider

provider = MeterProvider()
metrics.set_meter_provider(provider)
meter = metrics.get_meter("agent-manifest")

verifications = meter.create_counter(
    "agent_manifest.verifications",
    description="Total verification requests",
)
verification_latency = meter.create_histogram(
    "agent_manifest.verification.duration",
    unit="s",
    description="End-to-end verification latency",
)

# In your handler:
verifications.add(1, {"result": result.result.value})
verification_latency.record(duration, {"result": result.result.value})

groups:
  - name: agent-manifest
    rules:
      - alert: ManifestTamperingDetected
        expr: rate(agent_manifest_verifications_total{result="MISMATCH"}[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: Manifest artifact mismatch rate elevated
          description: Possible artifact tampering or key compromise. Rate = {{ $value }} req/s.

      - alert: ManifestRevocationSpike
        expr: rate(agent_manifest_verifications_total{result="REVOKED"}[5m]) > 0.5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: High revocation rate detected
          description: Multiple manifests being revoked. Rate = {{ $value }} req/s. Check for active incident.

      - alert: VerificationLatencyHigh
        expr: histogram_quantile(0.99, rate(agent_manifest_verification_duration_seconds_bucket[5m])) > 0.2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: Verification p99 latency above 200ms
          description: Check CRL endpoint availability and Rekor response times.