{
  "schemaVersion": 1,
  "slug": "transformer-decoder",
  "title": "One token at a time",
  "className": "TinyDecoder",
  "question": "How does a decoder form a next-token distribution?",
  "description": "Start with token IDs, add position, restrict attention to the visible prefix, and return a distribution over a small vocabulary.",
  "category": "Modern LLMs",
  "tags": [
    "Transformer",
    "Decoder",
    "Causal attention",
    "Language models"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "This is a teaching adaptation of the Transformer, not its full original translation model. It has no tokenizer or trained weights. Predictions are distributions over synthetic token IDs.",
  "openingStep": "causal",
  "provenance": {
    "capturedAt": "2026-09-17T05:43:00.236026+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "71dc54bd4d8dce7255cd674f020b03f48daba49120fab62e63b7aa232c1fb25b",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              6
            ],
            "dtype": "int64",
            "fill": "ones"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      6,
      32
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One CPU forward with integer token IDs, plus causality and probability checks using varied tokens. Untrained teaching decoder; no text-generation quality claim.",
    "numericalChecks": [
      "Changing future token IDs leaves earlier predictions unchanged",
      "Every attention head gives future positions exactly zero weight",
      "Attention rows and vocabulary distributions each sum to one"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,6"
      }
    },
    {
      "id": "embedding",
      "label": "embedding",
      "op": "Embedding",
      "kind": "layer",
      "parent": null,
      "params": {
        "num_embeddings": 32,
        "embedding_dim": 16
      },
      "source": {
        "line": 76,
        "endLine": 76
      }
    },
    {
      "id": "positions",
      "label": "positions",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 77,
        "endLine": 77
      }
    },
    {
      "id": "block",
      "label": "block",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 78,
        "endLine": 78
      }
    },
    {
      "id": "vocabulary",
      "label": "vocabulary",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 16,
        "out_features": 32
      },
      "source": {
        "line": 79,
        "endLine": 79
      }
    },
    {
      "id": "probabilities",
      "label": "probabilities",
      "op": "Softmax",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 80,
        "endLine": 80
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "python@positions#1",
      "label": "python@positions#1",
      "op": "Python",
      "kind": "layer",
      "parent": "positions",
      "params": {},
      "source": {
        "line": 15,
        "endLine": 15
      }
    },
    {
      "id": "python@positions#0",
      "label": "python@positions#0",
      "op": "Python",
      "kind": "layer",
      "parent": "positions",
      "params": {},
      "source": {
        "line": 14,
        "endLine": 14
      }
    },
    {
      "id": "mul@positions#0",
      "label": "mul@positions#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "positions",
      "params": {},
      "source": {
        "line": 16,
        "endLine": 16
      }
    },
    {
      "id": "stack@positions#0",
      "label": "stack@positions#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "positions",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 17,
        "endLine": 17
      }
    },
    {
      "id": "flatten@positions#0",
      "label": "flatten@positions#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "positions",
      "params": {
        "start_dim": -2
      },
      "source": {
        "line": 17,
        "endLine": 17
      }
    },
    {
      "id": "add@positions#0",
      "label": "add@positions#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "positions",
      "params": {},
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "block.attention",
      "label": "block.attention",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 61,
        "endLine": 61
      }
    },
    {
      "id": "add@block#0",
      "label": "Attention residual",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block",
      "params": {},
      "source": {
        "line": 68,
        "endLine": 68
      }
    },
    {
      "id": "block.norm_attention",
      "label": "block.norm_attention",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "block",
      "params": {
        "normalized_shape": 16
      },
      "source": {
        "line": 62,
        "endLine": 62
      }
    },
    {
      "id": "block.ffn",
      "label": "block.ffn",
      "op": "Module",
      "kind": "module",
      "parent": "block",
      "params": {},
      "source": {
        "line": 63,
        "endLine": 63
      }
    },
    {
      "id": "add@block#1",
      "label": "Feed-forward residual",
      "op": "torch.add",
      "kind": "layer",
      "parent": "block",
      "params": {},
      "source": {
        "line": 70,
        "endLine": 70
      }
    },
    {
      "id": "block.norm_ffn",
      "label": "block.norm_ffn",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "block",
      "params": {
        "normalized_shape": 16
      },
      "source": {
        "line": 64,
        "endLine": 64
      }
    },
    {
      "id": "block.attention.query",
      "label": "block.attention.query",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 31,
        "endLine": 31
      }
    },
    {
      "id": "python@block.attention#0",
      "label": "Split query heads · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 39,
        "endLine": 39
      }
    },
    {
      "id": "block.attention.key",
      "label": "block.attention.key",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 32,
        "endLine": 32
      }
    },
    {
      "id": "python@block.attention#1",
      "label": "Split key heads · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 40,
        "endLine": 40
      }
    },
    {
      "id": "block.attention.value",
      "label": "block.attention.value",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 33,
        "endLine": 33
      }
    },
    {
      "id": "python@block.attention#2",
      "label": "Split value heads · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 41,
        "endLine": 41
      }
    },
    {
      "id": "block.attention.scores",
      "label": "block.attention.scores",
      "op": "Module",
      "kind": "module",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 34,
        "endLine": 34
      }
    },
    {
      "id": "block.attention.softmax",
      "label": "block.attention.softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "python@block.attention#3",
      "label": "Mix and join heads · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention",
      "params": {},
      "source": {
        "line": 43,
        "endLine": 43
      }
    },
    {
      "id": "block.attention.project",
      "label": "block.attention.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 36,
        "endLine": 36
      }
    },
    {
      "id": "python@block.attention.scores#0",
      "label": "Scale attention scores · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.scores",
      "params": {},
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "python@block.attention.scores#1",
      "label": "Mask future positions · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "block.attention.scores",
      "params": {},
      "source": {
        "line": 25,
        "endLine": 25
      }
    },
    {
      "id": "block.ffn.expand",
      "label": "block.ffn.expand",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {
        "in_features": 16,
        "out_features": 64
      },
      "source": {
        "line": 50,
        "endLine": 50
      }
    },
    {
      "id": "block.ffn.relu",
      "label": "block.ffn.relu",
      "op": "ReLU",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {},
      "source": {
        "line": 51,
        "endLine": 51
      }
    },
    {
      "id": "block.ffn.project",
      "label": "block.ffn.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "block.ffn",
      "params": {
        "in_features": 64,
        "out_features": 16
      },
      "source": {
        "line": 52,
        "endLine": 52
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "python@positions#0",
      "target": "mul@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "python@positions#1",
      "target": "mul@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "mul@positions#0",
      "target": "stack@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "stack@positions#0",
      "target": "flatten@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "flatten@positions#0",
      "target": "add@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "python@block.attention.scores#0",
      "target": "python@block.attention.scores#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "block.attention.query",
      "target": "python@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "block.attention.key",
      "target": "python@block.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "block.attention.value",
      "target": "python@block.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "python@block.attention#0",
      "target": "python@block.attention.scores#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "python@block.attention#0",
      "target": "python@block.attention.scores#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "python@block.attention#0",
      "target": "block.attention.scores",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "python@block.attention#1",
      "target": "python@block.attention.scores#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "python@block.attention#1",
      "target": "python@block.attention.scores#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "python@block.attention#1",
      "target": "block.attention.scores",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "python@block.attention.scores#1",
      "target": "block.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "block.attention.scores",
      "target": "block.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "block.attention.softmax",
      "target": "python@block.attention#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "python@block.attention#2",
      "target": "python@block.attention#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "python@block.attention#3",
      "target": "block.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "block.ffn.expand",
      "target": "block.ffn.relu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "block.ffn.relu",
      "target": "block.ffn.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "block.attention.project",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "block.attention",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "add@block#0",
      "target": "block.norm_attention",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "block.norm_attention",
      "target": "block.ffn.expand",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "block.norm_attention",
      "target": "block.ffn",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "block.norm_attention",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "block.ffn.project",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "block.ffn",
      "target": "add@block#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "add@block#1",
      "target": "block.norm_ffn",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "input",
      "target": "embedding",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "embedding",
      "target": "python@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "embedding",
      "target": "python@positions#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "embedding",
      "target": "add@positions#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "embedding",
      "target": "positions",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "add@positions#0",
      "target": "block.attention.query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "add@positions#0",
      "target": "python@block.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "add@positions#0",
      "target": "block.attention.key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "add@positions#0",
      "target": "python@block.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "add@positions#0",
      "target": "block.attention.value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "add@positions#0",
      "target": "python@block.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-43",
      "source": "add@positions#0",
      "target": "python@block.attention#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-44",
      "source": "add@positions#0",
      "target": "add@block#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-45",
      "source": "positions",
      "target": "block",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-46",
      "source": "block.norm_ffn",
      "target": "vocabulary",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-47",
      "source": "block",
      "target": "vocabulary",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-48",
      "source": "vocabulary",
      "target": "probabilities",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-49",
      "source": "probabilities",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "tokens",
      "target": "embedding",
      "title": "Turn IDs into features",
      "note": "Each integer token ID indexes one of 32 learned vectors with 16 features. The positions block scales embeddings by √16 and adds sinusoidal position information."
    },
    {
      "id": "causal",
      "target": "block.attention.softmax",
      "title": "Restrict attention to the prefix",
      "note": "Four heads each have four features. Query–key scores are scaled by √4 and future positions are masked before softmax. The recorded rows below show the resulting zero weights."
    },
    {
      "id": "feed-forward",
      "target": "block.ffn",
      "title": "Transform each token's features",
      "note": "A 16 → 64 → 16 ReLU feed-forward network processes each position independently. Residual additions and LayerNorm surround the attention and feed-forward routes."
    },
    {
      "id": "predict",
      "target": "probabilities",
      "title": "Form a vocabulary distribution",
      "note": "The final projection creates 32 logits for every position. Softmax turns them into probabilities that sum to one; training would teach these outputs to predict the next token."
    }
  ],
  "milestone": {
    "name": "Transformer decoder",
    "year": 2017,
    "kind": "Architecture",
    "fidelity": "One reduced decoder-only block: width 16, four attention heads, FFN width 64, vocabulary 32. Keeps sinusoidal positions, causal masking and post-normalization; omits the original encoder, cross-attention and dropout.",
    "papers": [
      {
        "title": "Attention Is All You Need",
        "url": "https://arxiv.org/abs/1706.03762"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "A token can look back, but not ahead.",
    "description": "Recorded head-0 attention and vocabulary probabilities for the fixed token IDs [1, 2, 3, 4, 5, 6], using seed-0 untrained weights.",
    "controlLabel": "Query position",
    "cases": [
      {
        "id": "token-0",
        "label": "Query position 0",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 0 can consult positions 0 through 0. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              1,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.025921033092122455,
              0.04125577072333169,
              0.019280729378263906,
              0.06603918043332563,
              0.02476582830554817,
              0.01641311324043631,
              0.06969252752978246,
              0.15019487857439434
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 1
          },
          {
            "label": "Attention row sum",
            "value": 1
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 1
          }
        ]
      },
      {
        "id": "token-1",
        "label": "Query position 1",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 1 can consult positions 0 through 1. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              0.02233497503881591,
              0.9776650249611841,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.018538137538872988,
              0.0335681531897913,
              0.008095047180628643,
              0.03526194857138535,
              0.0293041558358887,
              0.02269042666961556,
              0.10577595352387788,
              0.07040253104150321
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 2
          },
          {
            "label": "Attention row sum",
            "value": 1
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 1
          }
        ]
      },
      {
        "id": "token-2",
        "label": "Query position 2",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 2 can consult positions 0 through 2. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              0.5099367074903837,
              0.008060307970987215,
              0.48200298453862905,
              0,
              0,
              0
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.023403421046833742,
              0.03459258601256063,
              0.03696927182279806,
              0.015686519297570395,
              0.015322580125723424,
              0.012502959862916815,
              0.01111306703971195,
              0.02624184052053954
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 3
          },
          {
            "label": "Attention row sum",
            "value": 0.9999999999999999
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 1
          }
        ]
      },
      {
        "id": "token-3",
        "label": "Query position 3",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 3 can consult positions 0 through 3. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              0.041964147069220706,
              0.012076369638673385,
              0.8890152194341486,
              0.05694426385795731,
              0,
              0
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.023144003980598914,
              0.014422629471549818,
              0.019566821006320446,
              0.04625198983105739,
              0.11939351250963462,
              0.04356744010438055,
              0.06841176389756196,
              0.03763608251009593
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 4
          },
          {
            "label": "Attention row sum",
            "value": 1
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 0.9999999999999998
          }
        ]
      },
      {
        "id": "token-4",
        "label": "Query position 4",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 4 can consult positions 0 through 4. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              0.7223768379821159,
              0.21696971883432192,
              0.012118540238537442,
              0.048519509459537334,
              0.00001539348548739001,
              0
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.014525092817356117,
              0.05610226170211578,
              0.03916504017802313,
              0.013153325148447,
              0.049608616957300455,
              0.016490656155215184,
              0.05296291046493026,
              0.03784821876356896
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 5
          },
          {
            "label": "Attention row sum",
            "value": 1
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 1.0000000000000002
          }
        ]
      },
      {
        "id": "token-5",
        "label": "Query position 5",
        "target": "block.attention.softmax",
        "note": "Head 0 at position 5 can consult positions 0 through 5. Later keys receive zero weight. Token IDs identify synthetic vocabulary entries; this model has no learned language ability.",
        "vectors": [
          {
            "label": "Attention over key positions 0–5",
            "values": [
              5.240008308328059e-7,
              0.00000887865764541971,
              0.00038877992599833884,
              0.0069491784055745465,
              0.969928249773367,
              0.022724389236583882
            ]
          },
          {
            "label": "Probabilities for vocabulary IDs 0–7 (of 32)",
            "values": [
              0.06271321166223129,
              0.030105346442901432,
              0.0236625648011329,
              0.026176426775969968,
              0.016649567795210485,
              0.025066678022755657,
              0.03899454137201595,
              0.032338340150309605
            ]
          }
        ],
        "metrics": [
          {
            "label": "Visible prefix length",
            "value": 6
          },
          {
            "label": "Attention row sum",
            "value": 1
          },
          {
            "label": "All 32 vocabulary probabilities sum to",
            "value": 0.9999999999999999
          }
        ]
      }
    ]
  },
  "revision": "11db495736878eab",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/transformer-decoder/11db495736878eab/graph.tensorviz.json",
      "sha256": "fe22fa7ab43528abd3183b1bcbeea03f2ab721336be49a4f0f70421d51567d04",
      "bytes": 82670
    },
    "model.py": {
      "url": "/models/transformer-decoder/11db495736878eab/model.py",
      "sha256": "71dc54bd4d8dce7255cd674f020b03f48daba49120fab62e63b7aa232c1fb25b",
      "bytes": 5703
    },
    "run.py": {
      "url": "/models/transformer-decoder/11db495736878eab/run.py",
      "sha256": "69050ee0e98ce22893e4db4e7771616859bf5a321718c88ffd1b6ae137bc80cc",
      "bytes": 619
    },
    "requirements.txt": {
      "url": "/models/transformer-decoder/11db495736878eab/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/transformer-decoder/11db495736878eab/preview.svg",
      "sha256": "dc169dd530669ab095029b94a936ed21c21374889bf121d98f037574567a9513",
      "bytes": 1950
    }
  }
}