{
  "schemaVersion": 1,
  "slug": "llama-decoder",
  "title": "Bring the modern decoder pieces together",
  "className": "TinyLlama",
  "question": "Where do RMSNorm, RoPE and SwiGLU meet in an actual decoder block?",
  "description": "Follow tokens through a reduced LLaMA-1-style decoder and inspect the activations where its familiar components connect.",
  "category": "Modern LLMs",
  "tags": [
    "LLaMA",
    "RMSNorm",
    "RoPE",
    "SwiGLU",
    "Causal decoder"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "A reduced, randomly initialized architecture; no tokenizer, pretrained LLaMA weights or generated-language quality. It models LLaMA-1-style multi-head attention. Hidden dimensions and training settings differ from the original models.",
  "openingStep": "pre-norm",
  "provenance": {
    "capturedAt": "2026-09-17T06:13:04.092484+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "66b559f494b1aea5da2a6dd8821e514150f696f08b43df73118108034614a783",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              6
            ],
            "dtype": "int64",
            "fill": "ones"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      6,
      32
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One untrained decoder layer with integer vocabulary IDs. Causality, prefix equivalence, norm preservation and identity-path checks; no language benchmark.",
    "numericalChecks": [
      "Later token changes leave earlier causal logits unchanged",
      "Zero attention/FFN output projections make the entire pre-norm block an identity",
      "RoPE preserves each head's query/key norm",
      "Prefix evaluation matches the corresponding full-sequence outputs"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,6,4"
      }
    },
    {
      "id": "embedding",
      "label": "embedding",
      "op": "Embedding",
      "kind": "layer",
      "parent": null,
      "params": {
        "num_embeddings": 32,
        "embedding_dim": 16
      },
      "source": {
        "line": 82,
        "endLine": 82
      }
    },
    {
      "id": "block",
      "label": "block",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 83,
        "endLine": 83
      }
    },
    {
      "id": "final_norm",
      "label": "final_norm",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 84,
        "endLine": 84
      }
    },
    {
      "id": "lm_head",
      "label": "lm_head",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 16,
        "out_features": 32,
        "bias": false
      },
      "source": {
        "line": 85,
        "endLine": 85
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "block.attention_norm",
      "label": "block.attention_norm",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 69,
        "endLine": 69
      }
    },
    {
      "id": "block.attention",
      "label": "block.attention",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 70,
        "endLine": 70
      }
    },
    {
      "id": "add@block#0",
      "label": "add@block#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block",
      "params": {},
      "source": {
        "line": 75,
        "endLine": 75
      }
    },
    {
      "id": "block.ffn_norm",
      "label": "block.ffn_norm",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 71,
        "endLine": 71
      }
    },
    {
      "id": "block.ffn",
      "label": "block.ffn",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 72,
        "endLine": 72
      }
    },
    {
      "id": "add@block#1",
      "label": "add@block#1",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block",
      "params": {},
      "source": {
        "line": 76,
        "endLine": 76
      }
    },
    {
      "id": "block.attention_norm.gain",
      "label": "block.attention_norm.gain",
      "op": "Parameter",
      "kind": "layer",
      "parent": "block.attention_norm",
      "params": {},
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "mul@block.attention_norm#0",
      "label": "mul@block.attention_norm#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention_norm",
      "params": {},
      "source": {
        "line": 16,
        "endLine": 16
      }
    },
    {
      "id": "python@block.attention#1",
      "label": "python@block.attention#1",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 49,
        "endLine": 49
      }
    },
    {
      "id": "block.attention.query",
      "label": "block.attention.query",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "block.attention.key",
      "label": "block.attention.key",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 36,
        "endLine": 36
      }
    },
    {
      "id": "block.attention.value",
      "label": "block.attention.value",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 37,
        "endLine": 37
      }
    },
    {
      "id": "python@block.attention#0",
      "label": "python@block.attention#0",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 47,
        "endLine": 47
      }
    },
    {
      "id": "block.attention.query_rope",
      "label": "block.attention.query_rope",
      "op": "Module",
      "kind": "module",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 38,
        "endLine": 38
      }
    },
    {
      "id": "block.attention.key_rope",
      "label": "block.attention.key_rope",
      "op": "Module",
      "kind": "module",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 39,
        "endLine": 39
      }
    },
    {
      "id": "div@block.attention#0",
      "label": "div@block.attention#0",
      "op": "torch.div",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 48,
        "endLine": 48
      }
    },
    {
      "id": "block.attention.softmax",
      "label": "block.attention.softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 40,
        "endLine": 40
      }
    },
    {
      "id": "block.attention.project",
      "label": "block.attention.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 41,
        "endLine": 41
      }
    },
    {
      "id": "python@block.attention.query_rope#2",
      "label": "python@block.attention.query_rope#2",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "python@block.attention.query_rope#1",
      "label": "python@block.attention.query_rope#1",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "python@block.attention.query_rope#0",
      "label": "python@block.attention.query_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 21,
        "endLine": 21
      }
    },
    {
      "id": "mul@block.attention.query_rope#0",
      "label": "mul@block.attention.query_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@block.attention.query_rope#1",
      "label": "mul@block.attention.query_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.query_rope#2",
      "label": "mul@block.attention.query_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "add@block.attention.query_rope#0",
      "label": "add@block.attention.query_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.query_rope#3",
      "label": "mul@block.attention.query_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.query_rope#4",
      "label": "mul@block.attention.query_rope#4",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "sub@block.attention.query_rope#0",
      "label": "sub@block.attention.query_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "stack@block.attention.query_rope#0",
      "label": "stack@block.attention.query_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "flatten@block.attention.query_rope#0",
      "label": "flatten@block.attention.query_rope#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "block.attention.query_rope",
      "params": {
        "start_dim": -2
      },
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "python@block.attention.key_rope#2",
      "label": "python@block.attention.key_rope#2",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "python@block.attention.key_rope#1",
      "label": "python@block.attention.key_rope#1",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "python@block.attention.key_rope#0",
      "label": "python@block.attention.key_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 21,
        "endLine": 21
      }
    },
    {
      "id": "mul@block.attention.key_rope#0",
      "label": "mul@block.attention.key_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@block.attention.key_rope#1",
      "label": "mul@block.attention.key_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.key_rope#2",
      "label": "mul@block.attention.key_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "add@block.attention.key_rope#0",
      "label": "add@block.attention.key_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.key_rope#3",
      "label": "mul@block.attention.key_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "mul@block.attention.key_rope#4",
      "label": "mul@block.attention.key_rope#4",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "sub@block.attention.key_rope#0",
      "label": "sub@block.attention.key_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "stack@block.attention.key_rope#0",
      "label": "stack@block.attention.key_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "flatten@block.attention.key_rope#0",
      "label": "flatten@block.attention.key_rope#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "block.attention.key_rope",
      "params": {
        "start_dim": -2
      },
      "source": {
        "line": 26,
        "endLine": 29
      }
    },
    {
      "id": "block.ffn_norm.gain",
      "label": "block.ffn_norm.gain",
      "op": "Parameter",
      "kind": "layer",
      "parent": "block.ffn_norm",
      "params": {},
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "mul@block.ffn_norm#0",
      "label": "mul@block.ffn_norm#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.ffn_norm",
      "params": {},
      "source": {
        "line": 16,
        "endLine": 16
      }
    },
    {
      "id": "block.ffn.gate",
      "label": "block.ffn.gate",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {
        "in_features": 16,
        "out_features": 40,
        "bias": false
      },
      "source": {
        "line": 57,
        "endLine": 57
      }
    },
    {
      "id": "block.ffn.up",
      "label": "block.ffn.up",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {
        "in_features": 16,
        "out_features": 40,
        "bias": false
      },
      "source": {
        "line": 58,
        "endLine": 58
      }
    },
    {
      "id": "block.ffn.silu",
      "label": "block.ffn.silu",
      "op": "SiLU",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {},
      "source": {
        "line": 59,
        "endLine": 59
      }
    },
    {
      "id": "mul@block.ffn#0",
      "label": "mul@block.ffn#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {},
      "source": {
        "line": 63,
        "endLine": 63
      }
    },
    {
      "id": "block.ffn.down",
      "label": "block.ffn.down",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {
        "in_features": 40,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 60,
        "endLine": 60
      }
    },
    {
      "id": "final_norm.gain",
      "label": "final_norm.gain",
      "op": "Parameter",
      "kind": "layer",
      "parent": "final_norm",
      "params": {},
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "mul@final_norm#0",
      "label": "mul@final_norm#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "final_norm",
      "params": {},
      "source": {
        "line": 16,
        "endLine": 16
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "block.attention_norm.gain",
      "target": "mul@block.attention_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "python@block.attention.query_rope#0",
      "target": "mul@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "python@block.attention.query_rope#1",
      "target": "mul@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "python@block.attention.query_rope#2",
      "target": "mul@block.attention.query_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "mul@block.attention.query_rope#0",
      "target": "mul@block.attention.query_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "python@block.attention.query_rope#2",
      "target": "mul@block.attention.query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "mul@block.attention.query_rope#0",
      "target": "mul@block.attention.query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "mul@block.attention.query_rope#4",
      "target": "sub@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "mul@block.attention.query_rope#3",
      "target": "sub@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "python@block.attention.query_rope#2",
      "target": "mul@block.attention.query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "mul@block.attention.query_rope#0",
      "target": "mul@block.attention.query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "python@block.attention.query_rope#2",
      "target": "mul@block.attention.query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "mul@block.attention.query_rope#0",
      "target": "mul@block.attention.query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "mul@block.attention.query_rope#2",
      "target": "add@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "mul@block.attention.query_rope#1",
      "target": "add@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "sub@block.attention.query_rope#0",
      "target": "stack@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "add@block.attention.query_rope#0",
      "target": "stack@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "stack@block.attention.query_rope#0",
      "target": "flatten@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "python@block.attention.key_rope#0",
      "target": "mul@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "python@block.attention.key_rope#1",
      "target": "mul@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "python@block.attention.key_rope#2",
      "target": "mul@block.attention.key_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "mul@block.attention.key_rope#0",
      "target": "mul@block.attention.key_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "python@block.attention.key_rope#2",
      "target": "mul@block.attention.key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "mul@block.attention.key_rope#0",
      "target": "mul@block.attention.key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "mul@block.attention.key_rope#4",
      "target": "sub@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "mul@block.attention.key_rope#3",
      "target": "sub@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "python@block.attention.key_rope#2",
      "target": "mul@block.attention.key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "mul@block.attention.key_rope#0",
      "target": "mul@block.attention.key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "python@block.attention.key_rope#2",
      "target": "mul@block.attention.key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "mul@block.attention.key_rope#0",
      "target": "mul@block.attention.key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "mul@block.attention.key_rope#2",
      "target": "add@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "mul@block.attention.key_rope#1",
      "target": "add@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "sub@block.attention.key_rope#0",
      "target": "stack@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "add@block.attention.key_rope#0",
      "target": "stack@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "stack@block.attention.key_rope#0",
      "target": "flatten@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "block.attention.query",
      "target": "python@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "block.attention.query",
      "target": "python@block.attention.query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "block.attention.query",
      "target": "python@block.attention.query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "block.attention.query",
      "target": "block.attention.query_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "block.attention.key",
      "target": "python@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "block.attention.key",
      "target": "python@block.attention.key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "block.attention.key",
      "target": "python@block.attention.key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-43",
      "source": "block.attention.key",
      "target": "block.attention.key_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-44",
      "source": "block.attention.value",
      "target": "python@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-45",
      "source": "flatten@block.attention.query_rope#0",
      "target": "div@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-46",
      "source": "block.attention.query_rope",
      "target": "div@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-47",
      "source": "flatten@block.attention.key_rope#0",
      "target": "div@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-48",
      "source": "block.attention.key_rope",
      "target": "div@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-49",
      "source": "div@block.attention#0",
      "target": "block.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-50",
      "source": "python@block.attention#1",
      "target": "block.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-51",
      "source": "block.attention.softmax",
      "target": "block.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-52",
      "source": "python@block.attention#0",
      "target": "block.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-53",
      "source": "block.ffn_norm.gain",
      "target": "mul@block.ffn_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-54",
      "source": "block.ffn.gate",
      "target": "block.ffn.silu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-55",
      "source": "block.ffn.silu",
      "target": "mul@block.ffn#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-56",
      "source": "block.ffn.up",
      "target": "mul@block.ffn#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-57",
      "source": "mul@block.ffn#0",
      "target": "block.ffn.down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-58",
      "source": "mul@block.attention_norm#0",
      "target": "block.attention.query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-59",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-60",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-61",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-62",
      "source": "mul@block.attention_norm#0",
      "target": "block.attention.key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-63",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-64",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-65",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention.key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-66",
      "source": "mul@block.attention_norm#0",
      "target": "block.attention.value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-67",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-68",
      "source": "mul@block.attention_norm#0",
      "target": "python@block.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-69",
      "source": "mul@block.attention_norm#0",
      "target": "block.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-70",
      "source": "block.attention_norm",
      "target": "block.attention",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-71",
      "source": "block.attention.project",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-72",
      "source": "block.attention",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-73",
      "source": "add@block#0",
      "target": "mul@block.ffn_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-74",
      "source": "add@block#0",
      "target": "block.ffn_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-75",
      "source": "mul@block.ffn_norm#0",
      "target": "block.ffn.gate",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-76",
      "source": "mul@block.ffn_norm#0",
      "target": "block.ffn.up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-77",
      "source": "block.ffn_norm",
      "target": "block.ffn",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-78",
      "source": "add@block#0",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-79",
      "source": "block.ffn.down",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-80",
      "source": "block.ffn",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-81",
      "source": "final_norm.gain",
      "target": "mul@final_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-82",
      "source": "input",
      "target": "embedding",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-83",
      "source": "embedding",
      "target": "mul@block.attention_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-84",
      "source": "embedding",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-85",
      "source": "embedding",
      "target": "block",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-86",
      "source": "add@block#1",
      "target": "mul@final_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-87",
      "source": "block",
      "target": "final_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-88",
      "source": "mul@final_norm#0",
      "target": "lm_head",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-89",
      "source": "final_norm",
      "target": "lm_head",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-90",
      "source": "lm_head",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "tokens",
      "target": "embedding",
      "title": "Map integer tokens into model features",
      "note": "The input contains six integer IDs from a vocabulary of 32. Each becomes a 16-feature vector. There are no learned absolute position embeddings; position enters through attention's RoPE."
    },
    {
      "id": "pre-norm",
      "target": "block.attention_norm",
      "title": "Normalize inside the residual branch",
      "note": "RMSNorm runs before attention. The original features bypass it on the residual path, so zeroing the branch's output projection leaves the block input available unchanged."
    },
    {
      "id": "rotary",
      "target": "block.attention.query_rope",
      "title": "Add position to causal attention",
      "note": "Four independent heads rotate their query/key pairs. The causal mask hides later tokens; values are mixed without rotation. This combines two different mechanisms: order information and future-token visibility."
    },
    {
      "id": "gated",
      "target": "block.ffn",
      "title": "Apply a second pre-norm and a SwiGLU branch",
      "note": "After attention rejoins the residual, another RMSNorm feeds a gated FFN. Its output is added to the running residual stream. A final RMSNorm and vocabulary head then produce 32 logits per token."
    }
  ],
  "milestone": {
    "name": "LLaMA architecture synthesis",
    "year": 2023,
    "kind": "Architecture",
    "fidelity": "One width-16 layer, four MHA heads, SwiGLU hidden width 40 and a 32-token vocabulary. Uses pre-RMSNorm and rotary Q/K, with a final RMSNorm and untied output head.",
    "papers": [
      {
        "title": "LLaMA: Open and Efficient Foundation Language Models",
        "url": "https://arxiv.org/abs/2302.13971"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "See the components in one decoder.",
    "description": "A small synthesis of LLaMA-1 architectural ingredients. It uses ordinary multi-head attention; later LLaMA-family models vary their head-sharing and training choices.",
    "controlLabel": "Activation checkpoint",
    "cases": [
      {
        "id": "embedding",
        "label": "Token embedding",
        "target": "embedding",
        "note": "These are recorded activations for the final token of the same six-token untrained decoder. RMSNorm controls feature scale inside each branch; residual additions preserve a direct input path. The vocabulary IDs have no language semantics.",
        "vectors": [
          {
            "label": "Final token · 16 features",
            "values": [
              1.8459428548812866,
              0.5134379863739014,
              -0.48757001757621765,
              -1.7857868671417236,
              -0.01136234775185585,
              1.2230252027511597,
              -0.38487744331359863,
              0.2809721827507019,
              1.7632919549942017,
              -0.6446139216423035,
              -0.6566305160522461,
              -0.004184153862297535,
              0.6146135330200195,
              0.022569775581359863,
              -1.1601743698120117,
              0.42933371663093567
            ]
          }
        ],
        "metrics": [
          {
            "label": "Feature RMS",
            "value": 0.9580375963466687
          },
          {
            "label": "Model width",
            "value": 16
          },
          {
            "label": "Attention heads",
            "value": 4
          }
        ]
      },
      {
        "id": "norm",
        "label": "Before attention: RMSNorm",
        "target": "block.attention_norm",
        "note": "These are recorded activations for the final token of the same six-token untrained decoder. RMSNorm controls feature scale inside each branch; residual additions preserve a direct input path. The vocabulary IDs have no language semantics.",
        "vectors": [
          {
            "label": "Final token · 16 features",
            "values": [
              1.9267947900230848,
              0.5359264695161966,
              -0.5089254887566583,
              -1.8640039818142704,
              -0.011860016355768666,
              1.2765934668543195,
              -0.40173499987448874,
              0.29327870926983957,
              1.8405237969248742,
              -0.672847885031964,
              -0.6853908039211769,
              -0.004367418981151683,
              0.64153348526968,
              0.023558327326958946,
              -1.2109897797545095,
              0.4481384492473546
            ]
          }
        ],
        "metrics": [
          {
            "label": "Feature RMS",
            "value": 0.9999994552408363
          },
          {
            "label": "Model width",
            "value": 16
          },
          {
            "label": "Attention heads",
            "value": 4
          }
        ]
      },
      {
        "id": "attention",
        "label": "After attention residual",
        "target": "block.attention",
        "note": "These are recorded activations for the final token of the same six-token untrained decoder. RMSNorm controls feature scale inside each branch; residual additions preserve a direct input path. The vocabulary IDs have no language semantics.",
        "vectors": [
          {
            "label": "Final token · 16 features",
            "values": [
              1.782397566570929,
              0.38412733529927107,
              -0.5343188660593108,
              -1.6134978283305053,
              0.18132015559442977,
              1.271392390279795,
              -0.3309789761453006,
              0.06451336751854825,
              1.7631419931926384,
              -0.6000220329971273,
              -0.7936437778362344,
              0.12646913241824476,
              0.5728971025749325,
              0.35126755995791914,
              -1.1800094219960506,
              0.4477696490172543
            ]
          }
        ],
        "metrics": [
          {
            "label": "Feature RMS",
            "value": 0.9399292643935143
          },
          {
            "label": "Model width",
            "value": 16
          },
          {
            "label": "Attention heads",
            "value": 4
          }
        ]
      },
      {
        "id": "ffn",
        "label": "Before SwiGLU: RMSNorm",
        "target": "block.ffn_norm",
        "note": "These are recorded activations for the final token of the same six-token untrained decoder. RMSNorm controls feature scale inside each branch; residual additions preserve a direct input path. The vocabulary IDs have no language semantics.",
        "vectors": [
          {
            "label": "Final token · 16 features",
            "values": [
              1.896309249367033,
              0.4086766232878675,
              -0.568466781386427,
              -1.7166152563724342,
              0.19290818984447033,
              1.3526461180603806,
              -0.3521315926265812,
              0.06863636813003327,
              1.875823066831805,
              -0.6383689881183333,
              -0.8443649524885929,
              0.13455167918879563,
              0.609510523871772,
              0.3737168045130876,
              -1.2554229332663933,
              0.47638626922657423
            ]
          }
        ],
        "metrics": [
          {
            "label": "Feature RMS",
            "value": 0.999999434048399
          },
          {
            "label": "Model width",
            "value": 16
          },
          {
            "label": "Attention heads",
            "value": 4
          }
        ]
      },
      {
        "id": "residual",
        "label": "After SwiGLU residual",
        "target": "block.ffn",
        "note": "These are recorded activations for the final token of the same six-token untrained decoder. RMSNorm controls feature scale inside each branch; residual additions preserve a direct input path. The vocabulary IDs have no language semantics.",
        "vectors": [
          {
            "label": "Final token · 16 features",
            "values": [
              1.6925145923360556,
              0.3326342906601701,
              -0.5842554292897966,
              -1.6258552739439613,
              0.24092541714949112,
              1.383114246033292,
              -0.42098633988148704,
              0.1529780920244319,
              1.6548177701232871,
              -0.5708168267049702,
              -0.8444525665545418,
              0.2022849611169636,
              0.6100495162311554,
              0.24500653528895533,
              -1.219728799477628,
              0.3863112848922764
            ]
          }
        ],
        "metrics": [
          {
            "label": "Feature RMS",
            "value": 0.9360531928192245
          },
          {
            "label": "Model width",
            "value": 16
          },
          {
            "label": "Attention heads",
            "value": 4
          }
        ]
      }
    ]
  },
  "revision": "14549022286b4d78",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/llama-decoder/14549022286b4d78/graph.tensorviz.json",
      "sha256": "7fa61eb0b63bbd811c0d662750850b30a2f659ad24e72dd2c77fc4c299984437",
      "bytes": 145454
    },
    "model.py": {
      "url": "/models/llama-decoder/14549022286b4d78/model.py",
      "sha256": "66b559f494b1aea5da2a6dd8821e514150f696f08b43df73118108034614a783",
      "bytes": 6513
    },
    "run.py": {
      "url": "/models/llama-decoder/14549022286b4d78/run.py",
      "sha256": "1fb81292ecaa43da6699d2f0035bb30120146043e69e345dfd42f448bfbb60c3",
      "bytes": 615
    },
    "requirements.txt": {
      "url": "/models/llama-decoder/14549022286b4d78/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/llama-decoder/14549022286b4d78/preview.svg",
      "sha256": "71f7f80381aaa1226e6fbb4f0452c513e362dc34979525aed4b0e2bfc68ffccb",
      "bytes": 1948
    }
  }
}