{
  "schemaVersion": 1,
  "slug": "self-attention",
  "title": "Inside self-attention",
  "className": "TokenMixer",
  "question": "How do queries, keys and values meet?",
  "description": "Trace one attention head from three projections to a weighted mixture, then follow the residual back to the output.",
  "category": "Attention",
  "tags": [
    "Attention",
    "Softmax",
    "Residual"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "One untrained, unmasked attention head over four 32-wide tokens; not a complete language model. TensorViz groups the score calculation and weighted value mixture into two Python nodes. Source shows the exact operations.",
  "openingStep": "weights",
  "provenance": {
    "capturedAt": "2026-09-17T01:50:58.662616+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "c8c21db7fcd9192b0f641cafd9e216a291d49926e230f137dc81a95adada0adb",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              4,
              32
            ],
            "dtype": "float32",
            "fill": "ones"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      4,
      32
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One CPU forward with float32 ones; untrained example, no accuracy claim. Installed VS Code workflow not separately verified for this example."
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,32"
      }
    },
    {
      "id": "attention",
      "label": "attention",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 30,
        "endLine": 30
      }
    },
    {
      "id": "norm",
      "label": "norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": null,
      "params": {
        "normalized_shape": 32
      },
      "source": {
        "line": 31,
        "endLine": 31
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "attention.query",
      "label": "attention.query",
      "op": "Linear",
      "kind": "layer",
      "parent": "attention",
      "params": {
        "in_features": 32,
        "out_features": 32
      },
      "source": {
        "line": 9,
        "endLine": 9
      }
    },
    {
      "id": "attention.key",
      "label": "attention.key",
      "op": "Linear",
      "kind": "layer",
      "parent": "attention",
      "params": {
        "in_features": 32,
        "out_features": 32
      },
      "source": {
        "line": 10,
        "endLine": 10
      }
    },
    {
      "id": "python@attention#0",
      "label": "Scores · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "attention",
      "params": {},
      "source": {
        "line": 19,
        "endLine": 19
      }
    },
    {
      "id": "attention.value",
      "label": "attention.value",
      "op": "Linear",
      "kind": "layer",
      "parent": "attention",
      "params": {
        "in_features": 32,
        "out_features": 32
      },
      "source": {
        "line": 11,
        "endLine": 11
      }
    },
    {
      "id": "attention.softmax",
      "label": "attention.softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": "attention",
      "params": {},
      "source": {
        "line": 12,
        "endLine": 12
      }
    },
    {
      "id": "python@attention#1",
      "label": "Mix values · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "attention",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "attention.project",
      "label": "attention.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "attention",
      "params": {
        "in_features": 32,
        "out_features": 32
      },
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "add@attention#0",
      "label": "add@attention#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "attention",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "attention.query",
      "target": "python@attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "attention.key",
      "target": "python@attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "python@attention#0",
      "target": "attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "attention.softmax",
      "target": "python@attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "attention.value",
      "target": "python@attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "python@attention#1",
      "target": "attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "attention.project",
      "target": "add@attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "input",
      "target": "attention.query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "input",
      "target": "attention.key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "input",
      "target": "attention.value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "input",
      "target": "add@attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "input",
      "target": "attention",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "add@attention#0",
      "target": "norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "attention",
      "target": "norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "norm",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "three-projections",
      "target": "attention.query",
      "title": "Three views of the same tokens",
      "note": "The same input enters separate query, key and value projections. Each preserves the 32-feature width. The expanded block shows their different paths."
    },
    {
      "id": "scores",
      "target": "python@attention#0",
      "title": "Compare queries with keys",
      "note": "The source multiplies Q by transposed K and scales by √32. Four tokens produce a 4 × 4 score matrix per example. These tensor expressions are grouped into this Python node."
    },
    {
      "id": "weights",
      "target": "attention.softmax",
      "title": "Turn scores into weights",
      "note": "Softmax normalizes each query's scores across the four keys. The next Python node multiplies those weights by V, producing four 32-wide mixtures."
    },
    {
      "id": "residual",
      "target": "add@attention#0",
      "title": "Keep a route for the original input",
      "note": "A final projection preserves the width so its result can be added to the original tokens. LayerNorm follows the attention block; the output is still [1, 4, 32]."
    }
  ],
  "revision": "d2b8dd93364c22a1",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/self-attention/d2b8dd93364c22a1/graph.tensorviz.json",
      "sha256": "6877c548e55bccda18e8d116b0b7f9723d1731c6ca507af2a667fb49e096cdde",
      "bytes": 33645
    },
    "model.py": {
      "url": "/models/self-attention/d2b8dd93364c22a1/model.py",
      "sha256": "c8c21db7fcd9192b0f641cafd9e216a291d49926e230f137dc81a95adada0adb",
      "bytes": 1221
    },
    "run.py": {
      "url": "/models/self-attention/d2b8dd93364c22a1/run.py",
      "sha256": "0bba6b5a61dbcd76694226262e3ff323f30c4717c7fbbb79068bb0a040222d83",
      "bytes": 402
    },
    "requirements.txt": {
      "url": "/models/self-attention/d2b8dd93364c22a1/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/self-attention/d2b8dd93364c22a1/preview.svg",
      "sha256": "5804c4f9b34c98d60462a1b5ba0b46f096ce8a15d0aa08c644d3360fc2c86d3e",
      "bytes": 2579
    }
  }
}