{
  "schemaVersion": 1,
  "slug": "vision-transformer",
  "title": "Turn an image into tokens",
  "className": "TinyViT",
  "question": "Where does an image become a sequence for a Transformer?",
  "description": "Inspect patch projection, learned positions and a class token, then compare its attention across a synthetic image.",
  "category": "Vision milestones",
  "tags": [
    "ViT",
    "Vision Transformer",
    "Image patches",
    "Attention"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "The paper title refers to one original patch-size configuration; this teaching model uses 4×4 patches to keep the example small. All weights are untrained. Attention maps are recorded probabilities, not saliency or evidence of recognition quality.",
  "openingStep": "patches",
  "provenance": {
    "capturedAt": "2026-09-17T06:18:49.600088+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "3d8cc36b80819465f0cf328570d80777db7ff486981777d425ee16eaa9ced676",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              3,
              16,
              16
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      3
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "Reduced ViT CPU forward with independent patch-projection equivalence, permutation/position tests and patch-local versus global influence checks.",
    "numericalChecks": [
      "Strided patch convolution equals independent flatten-and-linear projection",
      "Sixteen patch embeddings plus one class token form a seventeen-token sequence",
      "Without positions, patch permutation preserves the class output; fixed learned positions break this symmetry",
      "Changing one patch changes only its embedding, but global attention can change the class output"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,3,16,16"
      }
    },
    {
      "id": "patches",
      "label": "patches",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 59,
        "endLine": 59
      }
    },
    {
      "id": "class_token",
      "label": "class_token",
      "op": "Parameter",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 60,
        "endLine": 60
      }
    },
    {
      "id": "python@TinyViT#0",
      "label": "python@TinyViT#0",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 68,
        "endLine": 68
      }
    },
    {
      "id": "cat@TinyViT#0",
      "label": "cat@TinyViT#0",
      "op": "torch.cat",
      "kind": "layer",
      "parent": null,
      "params": {
        "dim": 1
      },
      "source": {
        "line": 69,
        "endLine": 69
      }
    },
    {
      "id": "positions",
      "label": "positions",
      "op": "Parameter",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 61,
        "endLine": 61
      }
    },
    {
      "id": "add@TinyViT#0",
      "label": "add@TinyViT#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 69,
        "endLine": 69
      }
    },
    {
      "id": "encoder",
      "label": "encoder",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 62,
        "endLine": 62
      }
    },
    {
      "id": "norm",
      "label": "norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": null,
      "params": {
        "normalized_shape": 16
      },
      "source": {
        "line": 63,
        "endLine": 63
      }
    },
    {
      "id": "head",
      "label": "head",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 16,
        "out_features": 3
      },
      "source": {
        "line": 64,
        "endLine": 64
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "patches.project",
      "label": "patches.project",
      "op": "Conv2d",
      "kind": "layer",
      "parent": "patches",
      "params": {
        "in_channels": 3,
        "out_channels": 16,
        "kernel_size": 4,
        "stride": 4
      },
      "source": {
        "line": 15,
        "endLine": 15
      }
    },
    {
      "id": "flatten@patches#0",
      "label": "flatten@patches#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "patches",
      "params": {
        "start_dim": 2
      },
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "python@patches#0",
      "label": "python@patches#0",
      "op": "Python",
      "kind": "layer",
      "parent": "patches",
      "params": {},
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "encoder.norm1",
      "label": "encoder.norm1",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "normalized_shape": 16
      },
      "source": {
        "line": 43,
        "endLine": 43
      }
    },
    {
      "id": "encoder.attention",
      "label": "encoder.attention",
      "op": "Module",
      "kind": "module",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 44,
        "endLine": 44
      }
    },
    {
      "id": "add@encoder#0",
      "label": "add@encoder#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 51,
        "endLine": 51
      }
    },
    {
      "id": "encoder.norm2",
      "label": "encoder.norm2",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "normalized_shape": 16
      },
      "source": {
        "line": 45,
        "endLine": 45
      }
    },
    {
      "id": "encoder.up",
      "label": "encoder.up",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "in_features": 16,
        "out_features": 32
      },
      "source": {
        "line": 46,
        "endLine": 46
      }
    },
    {
      "id": "encoder.gelu",
      "label": "encoder.gelu",
      "op": "GELU",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 47,
        "endLine": 47
      }
    },
    {
      "id": "encoder.down",
      "label": "encoder.down",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "in_features": 32,
        "out_features": 16
      },
      "source": {
        "line": 48,
        "endLine": 48
      }
    },
    {
      "id": "add@encoder#1",
      "label": "add@encoder#1",
      "op": "torch.add",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 52,
        "endLine": 52
      }
    },
    {
      "id": "encoder.attention.query",
      "label": "encoder.attention.query",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "python@encoder.attention#0",
      "label": "python@encoder.attention#0",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 32,
        "endLine": 32
      }
    },
    {
      "id": "encoder.attention.key",
      "label": "encoder.attention.key",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 25,
        "endLine": 25
      }
    },
    {
      "id": "python@encoder.attention#1",
      "label": "python@encoder.attention#1",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 33,
        "endLine": 33
      }
    },
    {
      "id": "div@encoder.attention#0",
      "label": "div@encoder.attention#0",
      "op": "torch.div",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "encoder.attention.value",
      "label": "encoder.attention.value",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 26,
        "endLine": 26
      }
    },
    {
      "id": "python@encoder.attention#2",
      "label": "python@encoder.attention#2",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 34,
        "endLine": 34
      }
    },
    {
      "id": "encoder.attention.softmax",
      "label": "encoder.attention.softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 27,
        "endLine": 27
      }
    },
    {
      "id": "encoder.attention.project",
      "label": "encoder.attention.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 16,
        "out_features": 16
      },
      "source": {
        "line": 28,
        "endLine": 28
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "patches.project",
      "target": "flatten@patches#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "flatten@patches#0",
      "target": "python@patches#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "encoder.attention.query",
      "target": "python@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "encoder.attention.key",
      "target": "python@encoder.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "encoder.attention.value",
      "target": "python@encoder.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "python@encoder.attention#0",
      "target": "div@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "python@encoder.attention#1",
      "target": "div@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "div@encoder.attention#0",
      "target": "encoder.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "encoder.attention.softmax",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "python@encoder.attention#2",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "encoder.norm1",
      "target": "encoder.attention.query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "encoder.norm1",
      "target": "python@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "encoder.norm1",
      "target": "encoder.attention.key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "encoder.norm1",
      "target": "python@encoder.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "encoder.norm1",
      "target": "encoder.attention.value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "encoder.norm1",
      "target": "python@encoder.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "encoder.norm1",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "encoder.norm1",
      "target": "encoder.attention",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "encoder.attention.project",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "encoder.attention",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "add@encoder#0",
      "target": "encoder.norm2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "encoder.norm2",
      "target": "encoder.up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "encoder.up",
      "target": "encoder.gelu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "encoder.gelu",
      "target": "encoder.down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "add@encoder#0",
      "target": "add@encoder#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "encoder.down",
      "target": "add@encoder#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "input",
      "target": "patches.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "input",
      "target": "patches",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "class_token",
      "target": "python@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "input",
      "target": "python@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "python@TinyViT#0",
      "target": "cat@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "python@patches#0",
      "target": "cat@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "patches",
      "target": "cat@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "cat@TinyViT#0",
      "target": "add@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "positions",
      "target": "add@TinyViT#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "add@TinyViT#0",
      "target": "encoder.norm1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "add@TinyViT#0",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "add@TinyViT#0",
      "target": "encoder",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "add@encoder#1",
      "target": "norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "encoder",
      "target": "norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "norm",
      "target": "head",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "head",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "patches",
      "target": "patches.project",
      "title": "Project non-overlapping RGB patches",
      "note": "A 4×4, stride-four convolution implements the same shared linear map as flattening each RGB patch. Sixteen patches each contain 48 input values and become 16-feature tokens."
    },
    {
      "id": "position",
      "target": "positions",
      "title": "Add a class token and learned positions",
      "note": "The learned class token precedes the sixteen image tokens. Seventeen position vectors are added so the encoder can distinguish patch locations. Without positions, reordering patches preserves the class output."
    },
    {
      "id": "attention",
      "target": "encoder.attention.softmax",
      "title": "Let the class token consult every patch",
      "note": "The two unmasked attention heads each compare all seventeen tokens. The heatmap below shows head 0's class-to-patch weights; its separate self-weight completes a row that sums to one."
    },
    {
      "id": "classify",
      "target": "head",
      "title": "Read the final class-token representation",
      "note": "After pre-normalized attention and GELU MLP residual branches, the class token is normalized and passed to a three-class head. The other tokens support that representation through attention."
    }
  ],
  "milestone": {
    "name": "Vision Transformer",
    "year": 2020,
    "kind": "Architecture",
    "fidelity": "A 16×16 RGB input becomes sixteen 4×4 patches with width 16, plus a class token. One pre-LN encoder, two attention heads, GELU MLP, learned positions and a linear three-class head.",
    "papers": [
      {
        "title": "An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale",
        "url": "https://arxiv.org/abs/2010.11929"
      }
    ]
  },
  "experiment": {
    "kind": "matrices",
    "title": "An image becomes a sequence of patches.",
    "description": "Recorded image-to-token conversion and class-token attention. A stride-four patch projection is equivalent to flattening each RGB patch and applying the same linear map. Learned positions preserve where each patch belongs.",
    "controlLabel": "Synthetic image",
    "cases": [
      {
        "id": "top-left",
        "label": "Bright patch at top left",
        "target": "patches",
        "note": "Move a bright 4×4 patch in a synthetic 16×16 image. The attention map shows head 0's class-token weights to the sixteen patch tokens, excluding its self-weight. These are untrained weights and are not a saliency explanation or object detector.",
        "matrices": [
          {
            "label": "Input image · red channel, exact pixels",
            "values": [
              [
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ]
            ]
          },
          {
            "label": "Class-to-patch attention · head 0",
            "values": [
              [
                0.06928419088319325,
                0.050439208577669814,
                0.05531670236301628,
                0.05799822467606256
              ],
              [
                0.060141283659673285,
                0.05389088655172201,
                0.05785637003319297,
                0.06021125655352544
              ],
              [
                0.059233013120851025,
                0.05507159274824081,
                0.06103305049542226,
                0.06262039880390745
              ],
              [
                0.05100755370790196,
                0.053389174840621896,
                0.054740370174549136,
                0.06470262604929915
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Bright patch embedding · 16 features",
            "values": [
              -0.016061378177255392,
              -0.7260191929526627,
              0.43272838479606435,
              1.146970379864797,
              0.5944235540227965,
              -0.19822190527338535,
              -0.36918258876539767,
              0.14066762826405466,
              -0.17082928074523807,
              0.13520527284345008,
              0.7760617169551551,
              -0.22844782704487443,
              1.1878711408935487,
              0.8733620627899654,
              -0.5369421225041151,
              0.049878431018441916
            ]
          },
          {
            "label": "Three untrained class logits",
            "values": [
              -0.09848402564951825,
              0.8794040492998403,
              0.5207556404259771
            ]
          }
        ],
        "metrics": [
          {
            "label": "Patch tokens",
            "value": 16
          },
          {
            "label": "Tokens including class token",
            "value": 17
          },
          {
            "label": "Class-token self-attention weight",
            "value": 0.07306409676115055
          },
          {
            "label": "Total class attention row weight",
            "value": 0.9999999999999998
          }
        ]
      },
      {
        "id": "middle",
        "label": "Bright patch near the middle",
        "target": "patches",
        "note": "Move a bright 4×4 patch in a synthetic 16×16 image. The attention map shows head 0's class-token weights to the sixteen patch tokens, excluding its self-weight. These are untrained weights and are not a saliency explanation or object detector.",
        "matrices": [
          {
            "label": "Input image · red channel, exact pixels",
            "values": [
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ]
            ]
          },
          {
            "label": "Class-to-patch attention · head 0",
            "values": [
              [
                0.057931638725673515,
                0.05026640633179098,
                0.05512719006350784,
                0.05779952561309896
              ],
              [
                0.059935242582117035,
                0.06824739773505524,
                0.0576581569572535,
                0.06000497575227904
              ],
              [
                0.05903008373345408,
                0.054882920182207855,
                0.060823954268581094,
                0.06240586439989355
              ],
              [
                0.05083280433958192,
                0.05320626615552882,
                0.05455283236075027,
                0.0644809580372335
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Bright patch embedding · 16 features",
            "values": [
              -0.016061378177255392,
              -0.7260191929526627,
              0.43272838479606435,
              1.146970379864797,
              0.5944235540227965,
              -0.19822190527338535,
              -0.36918258876539767,
              0.14066762826405466,
              -0.17082928074523807,
              0.13520527284345008,
              0.7760617169551551,
              -0.22844782704487443,
              1.1878711408935487,
              0.8733620627899654,
              -0.5369421225041151,
              0.049878431018441916
            ]
          },
          {
            "label": "Three untrained class logits",
            "values": [
              -0.09491749720176662,
              0.874794976506156,
              0.527605764370511
            ]
          }
        ],
        "metrics": [
          {
            "label": "Patch tokens",
            "value": 16
          },
          {
            "label": "Tokens including class token",
            "value": 17
          },
          {
            "label": "Class-token self-attention weight",
            "value": 0.07281378276199267
          },
          {
            "label": "Total class attention row weight",
            "value": 1
          }
        ]
      },
      {
        "id": "bottom-right",
        "label": "Bright patch at bottom right",
        "target": "patches",
        "note": "Move a bright 4×4 patch in a synthetic 16×16 image. The attention map shows head 0's class-token weights to the sixteen patch tokens, excluding its self-weight. These are untrained weights and are not a saliency explanation or object detector.",
        "matrices": [
          {
            "label": "Input image · red channel, exact pixels",
            "values": [
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1
              ],
              [
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                0,
                1,
                1,
                1,
                1
              ]
            ]
          },
          {
            "label": "Class-to-patch attention · head 0",
            "values": [
              [
                0.0584246994654985,
                0.05069422767500654,
                0.055596381919893904,
                0.058291461927817735
              ],
              [
                0.0604453561521658,
                0.05416335722742528,
                0.058148890072208345,
                0.06051568282683115
              ],
              [
                0.059532493425252575,
                0.05535003303839575,
                0.06134163173378728,
                0.06293700562681681
              ],
              [
                0.05126544634085066,
                0.05365910887724427,
                0.0550171358134924,
                0.07118358127550117
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Bright patch embedding · 16 features",
            "values": [
              -0.016061378177255392,
              -0.7260191929526627,
              0.43272838479606435,
              1.146970379864797,
              0.5944235540227965,
              -0.19822190527338535,
              -0.36918258876539767,
              0.14066762826405466,
              -0.17082928074523807,
              0.13520527284345008,
              0.7760617169551551,
              -0.22844782704487443,
              1.1878711408935487,
              0.8733620627899654,
              -0.5369421225041151,
              0.049878431018441916
            ]
          },
          {
            "label": "Three untrained class logits",
            "values": [
              -0.10053292223285593,
              0.8824995341436883,
              0.5283146670286265
            ]
          }
        ],
        "metrics": [
          {
            "label": "Patch tokens",
            "value": 16
          },
          {
            "label": "Tokens including class token",
            "value": 17
          },
          {
            "label": "Class-token self-attention weight",
            "value": 0.07343350660181171
          },
          {
            "label": "Total class attention row weight",
            "value": 0.9999999999999999
          }
        ]
      }
    ]
  },
  "revision": "0d36b50d0b83ec26",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/vision-transformer/0d36b50d0b83ec26/graph.tensorviz.json",
      "sha256": "211e6aeb23aaca387700f630daa832f70130863e8ad6c99601978a13b98243a8",
      "bytes": 73754
    },
    "model.py": {
      "url": "/models/vision-transformer/0d36b50d0b83ec26/model.py",
      "sha256": "3d8cc36b80819465f0cf328570d80777db7ff486981777d425ee16eaa9ced676",
      "bytes": 7579
    },
    "run.py": {
      "url": "/models/vision-transformer/0d36b50d0b83ec26/run.py",
      "sha256": "bc6189c612e316829e1499599cebd2054c836cd76690a9f08804f3b9f23be951",
      "bytes": 641
    },
    "requirements.txt": {
      "url": "/models/vision-transformer/0d36b50d0b83ec26/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/vision-transformer/0d36b50d0b83ec26/preview.svg",
      "sha256": "4f172911dc981bb1256351fd3ab20f0e2c229f7373abdd89ed4e61d89c07d819",
      "bytes": 1806
    }
  }
}