{
  "docVersion": 1,
  "id": "template-llm-serving-gateway",
  "name": "LLM serving gateway",
  "purpose": "Centralized model serving: authenticated, budget-governed traffic routed cheap-first through a cascade, with KV-cache reuse, continuous batching, hedged replicas and graceful degradation when the big model fails.",
  "nodes": {
    "clients": {
      "type": "users",
      "label": "Product teams",
      "x": 20,
      "y": 260,
      "runtime": "every AI feature in the org"
    },
    "gw": {
      "type": "gateway",
      "label": "LLM Gateway",
      "x": 270,
      "y": 260,
      "runtime": "authN · quotas · budget governor",
      "color": "blue",
      "patterns": [
        "stateless",
        "autoscale"
      ],
      "notes": "The single choke point where cost and safety live: per-team budgets, rate limits, audit of every prompt."
    },
    "router": {
      "type": "orchestrator",
      "label": "Model router",
      "x": 520,
      "y": 260,
      "runtime": "cheap-first cascade",
      "color": "purple",
      "notes": "Classifies difficulty and routes: 80% to the small model. Escalates on low confidence — and degrades the same path in reverse when the frontier model is down (fallback)."
    },
    "kv": {
      "type": "cache",
      "label": "Prefix / KV cache",
      "x": 520,
      "y": 470,
      "runtime": "shared system prompts",
      "color": "green"
    },
    "small": {
      "type": "agent",
      "label": "Small model",
      "x": 770,
      "y": 90,
      "runtime": "fast · handles ~80%",
      "patterns": [
        "replicated",
        "autoscale"
      ]
    },
    "batch": {
      "type": "gpu",
      "label": "Continuous batcher",
      "x": 770,
      "y": 300,
      "runtime": "in-flight batching · GPU-saturating"
    },
    "biga": {
      "type": "agent",
      "label": "Frontier model — replica A",
      "x": 1020,
      "y": 200,
      "runtime": "the expensive 20%"
    },
    "bigb": {
      "type": "agent",
      "label": "Frontier model — replica B",
      "x": 1020,
      "y": 400,
      "runtime": "hedge target",
      "patterns": [
        "standby"
      ]
    }
  },
  "edges": {
    "e1": {
      "from": "clients",
      "to": "gw",
      "fromSide": "right",
      "toSide": "left",
      "label": "prompt",
      "animated": true,
      "patterns": [
        "authenticated",
        "rate-limit"
      ]
    },
    "e2": {
      "from": "gw",
      "to": "router",
      "fromSide": "right",
      "toSide": "left",
      "label": "within budget",
      "animated": true
    },
    "e3": {
      "from": "router",
      "to": "kv",
      "fromSide": "bottom",
      "toSide": "top",
      "label": "prefix hit? skip prefill",
      "patterns": [
        "cache-aside"
      ]
    },
    "e4": {
      "from": "router",
      "to": "small",
      "fromSide": "right",
      "toSide": "left",
      "label": "easy 80% · also the fallback",
      "animated": true,
      "patterns": [
        "fallback"
      ]
    },
    "e5": {
      "from": "router",
      "to": "batch",
      "fromSide": "right",
      "toSide": "left",
      "label": "hard 20%"
    },
    "e6": {
      "from": "batch",
      "to": "biga",
      "fromSide": "right",
      "toSide": "left",
      "label": "batched decode",
      "patterns": [
        "batch"
      ]
    },
    "e7": {
      "from": "batch",
      "to": "bigb",
      "fromSide": "bottom",
      "toSide": "left",
      "label": "hedge at p95",
      "style": "dashed",
      "patterns": [
        "hedged"
      ]
    }
  },
  "boxes": {
    "b1": {
      "x": 740,
      "y": 20,
      "w": 470,
      "h": 520,
      "label": "model fleet — sized by the router, not by hope",
      "color": "purple",
      "opacity": 0.03
    }
  },
  "texts": {
    "t1": {
      "x": 20,
      "y": 440,
      "text": "the cascade is the cost model:\nsmall model $0.10/M handles 80%,\nfrontier $10/M handles 20% —\nblended cost drops ~7× vs frontier-only.",
      "color": "green",
      "size": 12
    }
  },
  "meta": {
    "createdAt": "2026-07-13T09:00:00.000Z",
    "modifiedAt": "2026-07-13T09:00:00.000Z"
  }
}