{
  "schemaVersion": 1,
  "slug": "latent-attention",
  "title": "Compress the cache, keep position separate",
  "className": "LatentAttention",
  "question": "What must MLA store to avoid reconstructing every key and value?",
  "description": "Trace joint KV compression and decoupled RoPE, then compare reconstructed attention with an absorbed-weight decoding calculation.",
  "category": "Attention systems",
  "tags": [
    "MLA",
    "DeepSeek-V2",
    "KV cache",
    "RoPE",
    "Low rank"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "An untrained MLA subsystem with reduced dimensions, not a complete DeepSeek model. The graph shows the explicit reconstructed forward; the CPU recipe separately verifies equivalent absorbed-weight inference. It includes the positional key in every cache estimate.",
  "openingStep": "compress",
  "provenance": {
    "capturedAt": "2026-09-17T06:13:00.308801+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "926d62027862b2121375cb2bfd54e4947208e89e13c7385545fa8cb502cdb656",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              6,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      6,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "Reduced MLA forward and independent absorbed-weight decoding for every prefix. Cache counts are analytical scalar counts, not speed or memory benchmarks.",
    "numericalChecks": [
      "Compressed-cache and absorbed-weight inference matches reconstructed attention for every prefix",
      "Cache stores three latent and two rotary-key values per token; query latents are not cached",
      "Causal outputs cannot change when only future inputs change",
      "The decoupled rotary path preserves pair norms and contributes to this example's output"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,6,8"
      }
    },
    {
      "id": "python@LatentAttention#5",
      "label": "python@LatentAttention#5",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 54,
        "endLine": 54
      }
    },
    {
      "id": "kv_latent",
      "label": "kv_latent",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 3,
        "bias": false
      },
      "source": {
        "line": 30,
        "endLine": 30
      }
    },
    {
      "id": "query_latent",
      "label": "query_latent",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 4,
        "bias": false
      },
      "source": {
        "line": 31,
        "endLine": 31
      }
    },
    {
      "id": "query_content",
      "label": "query_content",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 4,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 32,
        "endLine": 32
      }
    },
    {
      "id": "python@LatentAttention#0",
      "label": "python@LatentAttention#0",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 46,
        "endLine": 46
      }
    },
    {
      "id": "key_content",
      "label": "key_content",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 3,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 33,
        "endLine": 33
      }
    },
    {
      "id": "python@LatentAttention#1",
      "label": "python@LatentAttention#1",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 47,
        "endLine": 47
      }
    },
    {
      "id": "value_content",
      "label": "value_content",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 3,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 34,
        "endLine": 34
      }
    },
    {
      "id": "python@LatentAttention#2",
      "label": "python@LatentAttention#2",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 48,
        "endLine": 48
      }
    },
    {
      "id": "query_position",
      "label": "query_position",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 4,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "key_position",
      "label": "key_position",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 2,
        "bias": false
      },
      "source": {
        "line": 36,
        "endLine": 36
      }
    },
    {
      "id": "query_rope",
      "label": "query_rope",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 37,
        "endLine": 37
      }
    },
    {
      "id": "python@LatentAttention#3",
      "label": "python@LatentAttention#3",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 49,
        "endLine": 51
      }
    },
    {
      "id": "key_rope",
      "label": "key_rope",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 38,
        "endLine": 38
      }
    },
    {
      "id": "python@LatentAttention#4",
      "label": "python@LatentAttention#4",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 52,
        "endLine": 52
      }
    },
    {
      "id": "add@LatentAttention#0",
      "label": "add@LatentAttention#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 53,
        "endLine": 53
      }
    },
    {
      "id": "softmax",
      "label": "softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 39,
        "endLine": 39
      }
    },
    {
      "id": "output_projection",
      "label": "output_projection",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 40,
        "endLine": 40
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "python@query_rope#0",
      "label": "python@query_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 13,
        "endLine": 15
      }
    },
    {
      "id": "mul@query_rope#0",
      "label": "mul@query_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@query_rope#1",
      "label": "mul@query_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "add@query_rope#0",
      "label": "add@query_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@query_rope#2",
      "label": "mul@query_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@query_rope#3",
      "label": "mul@query_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "sub@query_rope#0",
      "label": "sub@query_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "stack@query_rope#0",
      "label": "stack@query_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "query_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "python@key_rope#0",
      "label": "python@key_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 13,
        "endLine": 15
      }
    },
    {
      "id": "mul@key_rope#0",
      "label": "mul@key_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@key_rope#1",
      "label": "mul@key_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "add@key_rope#0",
      "label": "add@key_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@key_rope#2",
      "label": "mul@key_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "mul@key_rope#3",
      "label": "mul@key_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "sub@key_rope#0",
      "label": "sub@key_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 23
      }
    },
    {
      "id": "stack@key_rope#0",
      "label": "stack@key_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "key_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 17,
        "endLine": 23
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "python@query_rope#0",
      "target": "mul@query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "python@query_rope#0",
      "target": "mul@query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "mul@query_rope#3",
      "target": "sub@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "mul@query_rope#2",
      "target": "sub@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "python@query_rope#0",
      "target": "mul@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "python@query_rope#0",
      "target": "mul@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "mul@query_rope#1",
      "target": "add@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "mul@query_rope#0",
      "target": "add@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "sub@query_rope#0",
      "target": "stack@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "add@query_rope#0",
      "target": "stack@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "python@key_rope#0",
      "target": "mul@key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "python@key_rope#0",
      "target": "mul@key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "mul@key_rope#3",
      "target": "sub@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "mul@key_rope#2",
      "target": "sub@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "python@key_rope#0",
      "target": "mul@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "python@key_rope#0",
      "target": "mul@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "mul@key_rope#1",
      "target": "add@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "mul@key_rope#0",
      "target": "add@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "sub@key_rope#0",
      "target": "stack@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "add@key_rope#0",
      "target": "stack@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "input",
      "target": "kv_latent",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "input",
      "target": "query_latent",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "query_latent",
      "target": "query_content",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "query_content",
      "target": "python@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "input",
      "target": "python@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "kv_latent",
      "target": "key_content",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "key_content",
      "target": "python@LatentAttention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "input",
      "target": "python@LatentAttention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "kv_latent",
      "target": "value_content",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "value_content",
      "target": "python@LatentAttention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "input",
      "target": "python@LatentAttention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "query_latent",
      "target": "query_position",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "query_position",
      "target": "python@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "query_position",
      "target": "mul@query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "query_position",
      "target": "mul@query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "query_position",
      "target": "mul@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "query_position",
      "target": "mul@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "query_position",
      "target": "query_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "input",
      "target": "python@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "input",
      "target": "mul@query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "input",
      "target": "mul@query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "input",
      "target": "mul@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-43",
      "source": "input",
      "target": "mul@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-44",
      "source": "input",
      "target": "query_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-45",
      "source": "stack@query_rope#0",
      "target": "python@LatentAttention#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-46",
      "source": "query_rope",
      "target": "python@LatentAttention#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-47",
      "source": "input",
      "target": "key_position",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-48",
      "source": "key_position",
      "target": "python@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-49",
      "source": "key_position",
      "target": "mul@key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-50",
      "source": "key_position",
      "target": "mul@key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-51",
      "source": "key_position",
      "target": "mul@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-52",
      "source": "key_position",
      "target": "mul@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-53",
      "source": "key_position",
      "target": "key_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-54",
      "source": "input",
      "target": "python@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-55",
      "source": "input",
      "target": "mul@key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-56",
      "source": "input",
      "target": "mul@key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-57",
      "source": "input",
      "target": "mul@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-58",
      "source": "input",
      "target": "mul@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-59",
      "source": "input",
      "target": "key_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-60",
      "source": "stack@key_rope#0",
      "target": "python@LatentAttention#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-61",
      "source": "key_rope",
      "target": "python@LatentAttention#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-62",
      "source": "python@LatentAttention#0",
      "target": "add@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-63",
      "source": "python@LatentAttention#1",
      "target": "add@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-64",
      "source": "python@LatentAttention#3",
      "target": "add@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-65",
      "source": "python@LatentAttention#4",
      "target": "add@LatentAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-66",
      "source": "input",
      "target": "python@LatentAttention#5",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-67",
      "source": "add@LatentAttention#0",
      "target": "softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-68",
      "source": "python@LatentAttention#5",
      "target": "softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-69",
      "source": "softmax",
      "target": "output_projection",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-70",
      "source": "python@LatentAttention#2",
      "target": "output_projection",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-71",
      "source": "input",
      "target": "output_projection",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-72",
      "source": "output_projection",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "compress",
      "target": "kv_latent",
      "title": "Create one shared latent for K and V",
      "note": "An 8-to-3 projection produces the joint KV latent. Content-key and value up-projections reconstruct four heads from it in this explicit forward."
    },
    {
      "id": "query",
      "target": "query_content",
      "title": "Compress and expand the current query",
      "note": "The query has its own 8-to-4 down-projection and a four-to-eight content up-projection. Query latents are used for current queries and do not become part of the prefix KV cache."
    },
    {
      "id": "position",
      "target": "key_rope",
      "title": "Cache a separate positional key",
      "note": "RoPE acts on an additional shared width-two key and per-head positional queries. Keeping these features separate lets content-key weights be absorbed into the query calculation without moving them across token-dependent rotations."
    },
    {
      "id": "absorb",
      "target": "output_projection",
      "title": "Absorb value reconstruction into the output",
      "note": "During compact inference, attention mixes latent vectors. Combining value-up and output-projection weights then gives the same result without expanding cached values. Compare this independent calculation below."
    }
  ],
  "milestone": {
    "name": "Multi-head Latent Attention",
    "year": 2024,
    "kind": "Component",
    "fidelity": "Four heads with content/value width two and rotary width two; KV latent width three and query latent width four. Includes the shared decoupled rotary key from DeepSeek-V2 equations 9–19.",
    "papers": [
      {
        "title": "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model",
        "url": "https://arxiv.org/abs/2405.04434"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "Cache a latent plus the positional key.",
    "description": "Recorded decoding comparisons for 1, 3 and 6 prefix tokens. Expanded storage counts 8 content-key + 8 value + 2 shared rotary-key values per token; the compact cache needs 3 + 2. These teaching dimensions are not DeepSeek-V2's production dimensions.",
    "controlLabel": "Prefix length",
    "cases": [
      {
        "id": "prefix-1",
        "label": "1 cached token",
        "target": "kv_latent",
        "note": "The cache retains the joint KV latent and a separate rotated key. The content-key up-projection is absorbed into the query calculation; the value up-projection is absorbed into the output projection. The positional path stays separate because token-dependent rotations cannot be freely reordered with learned matrices.",
        "vectors": [
          {
            "label": "KV latent · final cached token",
            "values": [
              0.5289286386768289,
              -0.5837054691333015,
              0.21774575875425659
            ]
          },
          {
            "label": "Rotary key · final cached token",
            "values": [
              -0.7281829658503451,
              -0.267078965381432
            ]
          },
          {
            "label": "Reconstructed attention output",
            "values": [
              0.14386388222362784,
              0.10056909900067523,
              -0.3434470255791625,
              -0.03132911136333658,
              -0.11722114877828872,
              0.05268442274285208,
              -0.01363043207637249,
              -0.08805563320309245
            ]
          },
          {
            "label": "Absorbed-weight output",
            "values": [
              0.14386388222362784,
              0.10056909900067523,
              -0.3434470255791625,
              -0.031329111363336576,
              -0.11722114877828872,
              0.05268442274285208,
              -0.013630432076372493,
              -0.08805563320309244
            ]
          }
        ],
        "metrics": [
          {
            "label": "Compact cache · scalar values",
            "value": 5
          },
          {
            "label": "Expanded content K/V + shared rotary key · scalar values",
            "value": 18
          },
          {
            "label": "Maximum output difference",
            "value": 1.3877787807814457e-17
          }
        ]
      },
      {
        "id": "prefix-3",
        "label": "3 cached tokens",
        "target": "kv_latent",
        "note": "The cache retains the joint KV latent and a separate rotated key. The content-key up-projection is absorbed into the query calculation; the value up-projection is absorbed into the output projection. The positional path stays separate because token-dependent rotations cannot be freely reordered with learned matrices.",
        "vectors": [
          {
            "label": "KV latent · final cached token",
            "values": [
              -0.02780165472797927,
              0.015460745308090627,
              -0.1984789600912455
            ]
          },
          {
            "label": "Rotary key · final cached token",
            "values": [
              0.07457904280575867,
              -0.1022349319542943
            ]
          },
          {
            "label": "Reconstructed attention output",
            "values": [
              0.055412680857138914,
              0.14504390972268869,
              -0.20036087748121356,
              0.000676401913339314,
              -0.017525903907166098,
              -0.016993711739130155,
              0.014551908110502013,
              -0.04815015031453829
            ]
          },
          {
            "label": "Absorbed-weight output",
            "values": [
              0.05541268085713891,
              0.14504390972268869,
              -0.20036087748121356,
              0.0006764019133393329,
              -0.01752590390716611,
              -0.01699371173913016,
              0.014551908110502016,
              -0.04815015031453829
            ]
          }
        ],
        "metrics": [
          {
            "label": "Compact cache · scalar values",
            "value": 15
          },
          {
            "label": "Expanded content K/V + shared rotary key · scalar values",
            "value": 54
          },
          {
            "label": "Maximum output difference",
            "value": 1.8865117801247777e-17
          }
        ]
      },
      {
        "id": "prefix-6",
        "label": "6 cached tokens",
        "target": "kv_latent",
        "note": "The cache retains the joint KV latent and a separate rotated key. The content-key up-projection is absorbed into the query calculation; the value up-projection is absorbed into the output projection. The positional path stays separate because token-dependent rotations cannot be freely reordered with learned matrices.",
        "vectors": [
          {
            "label": "KV latent · final cached token",
            "values": [
              -0.19908599241925576,
              -0.07575139978063039,
              0.07169747709683116
            ]
          },
          {
            "label": "Rotary key · final cached token",
            "values": [
              0.2083996207854994,
              0.044331236291429627
            ]
          },
          {
            "label": "Reconstructed attention output",
            "values": [
              0.026221660382326113,
              0.13984915056230562,
              -0.13882107493978346,
              0.02207917746520416,
              0.009273062786950396,
              -0.03218115056801804,
              0.022071498565130323,
              -0.027426152272238692
            ]
          },
          {
            "label": "Absorbed-weight output",
            "values": [
              0.02622166038232611,
              0.13984915056230565,
              -0.13882107493978346,
              0.022079177465204148,
              0.009273062786950387,
              -0.032181150568018034,
              0.022071498565130333,
              -0.027426152272238682
            ]
          }
        ],
        "metrics": [
          {
            "label": "Compact cache · scalar values",
            "value": 30
          },
          {
            "label": "Expanded content K/V + shared rotary key · scalar values",
            "value": 108
          },
          {
            "label": "Maximum output difference",
            "value": 2.7755575615628914e-17
          }
        ]
      }
    ]
  },
  "revision": "a52b24527129a0bd",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/latent-attention/a52b24527129a0bd/graph.tensorviz.json",
      "sha256": "83c3ebb0510746d4ef073fab2304c15e22d409d9a64276371cb00f41beb8dd6f",
      "bytes": 89069
    },
    "model.py": {
      "url": "/models/latent-attention/a52b24527129a0bd/model.py",
      "sha256": "926d62027862b2121375cb2bfd54e4947208e89e13c7385545fa8cb502cdb656",
      "bytes": 7754
    },
    "run.py": {
      "url": "/models/latent-attention/a52b24527129a0bd/run.py",
      "sha256": "57b40a0ee00aba882469b7ef8ae1bddafb9836b424212d96337d9cd3fb7e8a77",
      "bytes": 654
    },
    "requirements.txt": {
      "url": "/models/latent-attention/a52b24527129a0bd/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/latent-attention/a52b24527129a0bd/preview.svg",
      "sha256": "ab01119581fcb89296e955784e8b7600644fbb2398e247c51737cb6db4d44e56",
      "bytes": 2206
    }
  }
}