{
  "schemaVersion": 1,
  "slug": "rope",
  "title": "Put position into attention",
  "className": "RotaryAttention",
  "question": "Where does token position enter the attention score?",
  "description": "Rotate query and key feature pairs, compare relative positions, and follow the unchanged value path to the output.",
  "category": "Modern LLMs",
  "tags": [
    "RoPE",
    "RoFormer",
    "Rotary position embeddings",
    "Attention"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "This component is not a trained RoFormer model. The diagram visualizes only the first feature pair; other pairs use slower frequencies. Relative-position algebra does not guarantee context-length extrapolation in a trained model.",
  "openingStep": "rotate-query",
  "provenance": {
    "capturedAt": "2026-09-17T05:53:58.907442+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "a5a791e4b252ae091485fec653a9bd53798407bb6df40d663255744f24a53b7e",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              6,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      6,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One unmasked CPU attention forward plus norm, relative-position and unchanged-value checks. Numerical rotations use fixed synthetic Q/K vectors in float64.",
    "numericalChecks": [
      "Every paired rotation preserves the vector norm",
      "Shifting both positions equally preserves the Q/K dot product for fixed content",
      "Changing relative position changes the score in this example",
      "Zero Q/K yields a uniform mixture of unrotated values"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,6,8"
      }
    },
    {
      "id": "query",
      "label": "query",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 31,
        "endLine": 31
      }
    },
    {
      "id": "key",
      "label": "key",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 32,
        "endLine": 32
      }
    },
    {
      "id": "value",
      "label": "value",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 33,
        "endLine": 33
      }
    },
    {
      "id": "query_rope",
      "label": "query_rope",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 34,
        "endLine": 34
      }
    },
    {
      "id": "key_rope",
      "label": "key_rope",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "div@RotaryAttention#0",
      "label": "div@RotaryAttention#0",
      "op": "torch.div",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 43,
        "endLine": 43
      }
    },
    {
      "id": "softmax",
      "label": "softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 36,
        "endLine": 36
      }
    },
    {
      "id": "python@RotaryAttention#0",
      "label": "python@RotaryAttention#0",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 45,
        "endLine": 45
      }
    },
    {
      "id": "project",
      "label": "project",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 37,
        "endLine": 37
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "python@query_rope#1",
      "label": "python@query_rope#1",
      "op": "Python",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 20,
        "endLine": 20
      }
    },
    {
      "id": "python@query_rope#0",
      "label": "python@query_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "mul@query_rope#0",
      "label": "mul@query_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 19,
        "endLine": 19
      }
    },
    {
      "id": "mul@query_rope#3",
      "label": "mul@query_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@query_rope#4",
      "label": "mul@query_rope#4",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "add@query_rope#0",
      "label": "add@query_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@query_rope#1",
      "label": "mul@query_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "mul@query_rope#2",
      "label": "mul@query_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "sub@query_rope#0",
      "label": "sub@query_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "query_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "stack@query_rope#0",
      "label": "stack@query_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "query_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "flatten@query_rope#0",
      "label": "flatten@query_rope#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "query_rope",
      "params": {
        "start_dim": -2
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "python@key_rope#1",
      "label": "python@key_rope#1",
      "op": "Python",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 20,
        "endLine": 20
      }
    },
    {
      "id": "python@key_rope#0",
      "label": "python@key_rope#0",
      "op": "Python",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "mul@key_rope#0",
      "label": "mul@key_rope#0",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 19,
        "endLine": 19
      }
    },
    {
      "id": "mul@key_rope#3",
      "label": "mul@key_rope#3",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@key_rope#4",
      "label": "mul@key_rope#4",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "add@key_rope#0",
      "label": "add@key_rope#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 23,
        "endLine": 23
      }
    },
    {
      "id": "mul@key_rope#1",
      "label": "mul@key_rope#1",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "mul@key_rope#2",
      "label": "mul@key_rope#2",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "sub@key_rope#0",
      "label": "sub@key_rope#0",
      "op": "torch.sub",
      "kind": "layer",
      "parent": "key_rope",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "stack@key_rope#0",
      "label": "stack@key_rope#0",
      "op": "torch.stack",
      "kind": "layer",
      "parent": "key_rope",
      "params": {
        "dim": -1
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "flatten@key_rope#0",
      "label": "flatten@key_rope#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": "key_rope",
      "params": {
        "start_dim": -2
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "python@query_rope#0",
      "target": "mul@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "python@query_rope#1",
      "target": "mul@query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "mul@query_rope#0",
      "target": "mul@query_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "python@query_rope#1",
      "target": "mul@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "mul@query_rope#0",
      "target": "mul@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "mul@query_rope#2",
      "target": "sub@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "mul@query_rope#1",
      "target": "sub@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "python@query_rope#1",
      "target": "mul@query_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "mul@query_rope#0",
      "target": "mul@query_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "python@query_rope#1",
      "target": "mul@query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "mul@query_rope#0",
      "target": "mul@query_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "mul@query_rope#4",
      "target": "add@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "mul@query_rope#3",
      "target": "add@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "sub@query_rope#0",
      "target": "stack@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "add@query_rope#0",
      "target": "stack@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "stack@query_rope#0",
      "target": "flatten@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "python@key_rope#0",
      "target": "mul@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "python@key_rope#1",
      "target": "mul@key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "mul@key_rope#0",
      "target": "mul@key_rope#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "python@key_rope#1",
      "target": "mul@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "mul@key_rope#0",
      "target": "mul@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "mul@key_rope#2",
      "target": "sub@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "mul@key_rope#1",
      "target": "sub@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "python@key_rope#1",
      "target": "mul@key_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "mul@key_rope#0",
      "target": "mul@key_rope#4",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "python@key_rope#1",
      "target": "mul@key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "mul@key_rope#0",
      "target": "mul@key_rope#3",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "mul@key_rope#4",
      "target": "add@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "mul@key_rope#3",
      "target": "add@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "sub@key_rope#0",
      "target": "stack@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "add@key_rope#0",
      "target": "stack@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "stack@key_rope#0",
      "target": "flatten@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "input",
      "target": "query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "query",
      "target": "python@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "query",
      "target": "mul@query_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "query",
      "target": "python@query_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "query",
      "target": "query_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "input",
      "target": "key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "key",
      "target": "python@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "key",
      "target": "mul@key_rope#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "key",
      "target": "python@key_rope#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "key",
      "target": "key_rope",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-43",
      "source": "input",
      "target": "value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-44",
      "source": "flatten@query_rope#0",
      "target": "div@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-45",
      "source": "query_rope",
      "target": "div@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-46",
      "source": "flatten@key_rope#0",
      "target": "div@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-47",
      "source": "key_rope",
      "target": "div@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-48",
      "source": "div@RotaryAttention#0",
      "target": "softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-49",
      "source": "softmax",
      "target": "python@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-50",
      "source": "value",
      "target": "python@RotaryAttention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-51",
      "source": "python@RotaryAttention#0",
      "target": "project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-52",
      "source": "project",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "project",
      "target": "query",
      "title": "Start with content projections",
      "note": "The input has six tokens with eight features. Independent learned projections produce Q, K and V; position enters after Q and K have been projected."
    },
    {
      "id": "rotate-query",
      "target": "query_rope",
      "title": "Rotate query feature pairs",
      "note": "Each adjacent pair is rotated by position × frequency. The four frequencies are 1, 0.1, 0.01 and 0.001 radians per position, preserving the query norm."
    },
    {
      "id": "rotate-key",
      "target": "key_rope",
      "title": "Rotate keys with the same rule",
      "note": "When rotated Q and K meet, their dot product depends on relative position. Shift both positions together in the recorded comparison to see the score stay fixed."
    },
    {
      "id": "values",
      "target": "value",
      "title": "Keep values on their content path",
      "note": "This standard RoPE attention variant leaves V unrotated. Softmax of the scaled Q/K scores chooses how to mix these value vectors before the output projection."
    }
  ],
  "milestone": {
    "name": "RoPE / RoFormer",
    "year": 2021,
    "kind": "Component",
    "fidelity": "One unmasked attention head with width 8 and four paired rotations using base 10,000. Rotates Q/K only; isolates position encoding from causal masking.",
    "papers": [
      {
        "title": "RoFormer: Enhanced Transformer with Rotary Position Embedding",
        "url": "https://arxiv.org/abs/2104.09864"
      }
    ]
  },
  "experiment": {
    "kind": "rotary",
    "title": "Move position, keep content fixed.",
    "description": "Recorded float64 rotations of the same synthetic Q/K vectors. The diagram shows their first feature pair (one radian per position); the score uses all four pairs.",
    "controlLabel": "Token positions",
    "cases": [
      {
        "id": "same-position",
        "label": "Both positions 0",
        "note": "At position zero, all rotation angles are zero. The query and key vectors remain unchanged.",
        "target": "query_rope",
        "rotation": {
          "queryPosition": 0,
          "keyPosition": 0,
          "query": [
            1,
            0.5
          ],
          "key": [
            0.2,
            1
          ],
          "rotatedQuery": [
            1,
            0.5
          ],
          "rotatedKey": [
            0.2,
            1
          ]
        },
        "vectors": [
          {
            "label": "Rotated Q · all 8 features",
            "values": [
              1,
              0.5,
              0.2,
              -0.3,
              0.7,
              -0.1,
              0.2,
              0.8
            ]
          },
          {
            "label": "Rotated K · all 8 features",
            "values": [
              0.2,
              1,
              -0.4,
              0.6,
              0.1,
              0.3,
              -0.5,
              0.4
            ]
          }
        ],
        "metrics": [
          {
            "label": "Key position − query position",
            "value": 0
          },
          {
            "label": "Scaled Q/K score · all 8 features",
            "value": 0.24748737341529164
          },
          {
            "label": "Query norm",
            "value": 1.6
          }
        ]
      },
      {
        "id": "gap-two",
        "label": "Query 1 · key 3",
        "note": "The key is two positions after the query. Each feature pair rotates at its own frequency.",
        "target": "query_rope",
        "rotation": {
          "queryPosition": 1,
          "keyPosition": 3,
          "query": [
            1,
            0.5
          ],
          "key": [
            0.2,
            1
          ],
          "rotatedQuery": [
            0.11956681346419151,
            1.1116221377419664
          ],
          "rotatedKey": [
            -0.3391185073799563,
            -0.961768494988472
          ]
        },
        "vectors": [
          {
            "label": "Rotated Q · all 8 features",
            "values": [
              0.11956681346419151,
              1.1116221377419664,
              0.22895085804965362,
              -0.2785345662540421,
              0.7009649836250823,
              -0.09299511670774986,
              0.19919990013334166,
              0.8001995999667
            ]
          },
          {
            "label": "Rotated K · all 8 features",
            "values": [
              -0.3391185073799563,
              -0.961768494988472,
              -0.5594467196470462,
              0.4549938108108277,
              0.09095635331415007,
              0.3028645601449458,
              -0.5011977482016883,
              0.39849820225134897
            ]
          }
        ],
        "metrics": [
          {
            "label": "Key position − query position",
            "value": 2
          },
          {
            "label": "Scaled Q/K score · all 8 features",
            "value": -0.392393325612076
          },
          {
            "label": "Query norm",
            "value": 1.6
          }
        ]
      },
      {
        "id": "shift-both",
        "label": "Shift both by 10: query 11 · key 13",
        "note": "The absolute angles change, but the relative offset remains two. For these fixed query/key contents, the score matches query 1 / key 3.",
        "target": "query_rope",
        "rotation": {
          "queryPosition": 11,
          "keyPosition": 13,
          "query": [
            1,
            0.5
          ],
          "key": [
            0.2,
            1
          ],
          "rotatedQuery": [
            0.5044208012634025,
            -0.9977773575566781
          ],
          "rotatedKey": [
            -0.23867768053660166,
            0.9914801888155244
          ]
        },
        "vectors": [
          {
            "label": "Rotated Q · all 8 features",
            "values": [
              0.5044208012634025,
              -0.9977773575566781,
              0.3580814323035461,
              0.04216263558461392,
              0.7067470986534051,
              -0.022550799209647324,
              0.19118807758760084,
              0.8021515561216331
            ]
          },
          {
            "label": "Rotated K · all 8 features",
            "values": [
              -0.23867768053660166,
              0.9914801888155244,
              -0.6851344427001508,
              -0.22492397699212482,
              0.06026594658557036,
              0.3104319823764059,
              -0.5051576041295884,
              0.3934663835578003
            ]
          }
        ],
        "metrics": [
          {
            "label": "Key position − query position",
            "value": 2
          },
          {
            "label": "Scaled Q/K score · all 8 features",
            "value": -0.392393325612076
          },
          {
            "label": "Query norm",
            "value": 1.5999999999999999
          }
        ]
      },
      {
        "id": "gap-four",
        "label": "Query 1 · key 5",
        "note": "Changing the relative offset to four changes the score. RoPE supplies position dependence; this does not prove extrapolation quality.",
        "target": "query_rope",
        "rotation": {
          "queryPosition": 1,
          "keyPosition": 5,
          "query": [
            1,
            0.5
          ],
          "key": [
            0.2,
            1
          ],
          "rotatedQuery": [
            0.11956681346419151,
            1.1116221377419664
          ],
          "rotatedKey": [
            1.0156567117557838,
            0.09187733053059854
          ]
        },
        "vectors": [
          {
            "label": "Rotated Q · all 8 features",
            "values": [
              0.11956681346419151,
              1.1116221377419664,
              0.22895085804965362,
              -0.2785345662540421,
              0.7009649836250823,
              -0.09299511670774986,
              0.19919990013334166,
              0.8001995999667
            ]
          },
          {
            "label": "Rotated K · all 8 features",
            "values": [
              1.0156567117557838,
              0.09187733053059854,
              -0.6386883479186709,
              0.3347793216925424,
              0.08488127525829314,
              0.30462299504555773,
              -0.5019937416796979,
              0.3974950104270703
            ]
          }
        ],
        "metrics": [
          {
            "label": "Key position − query position",
            "value": 4
          },
          {
            "label": "Scaled Q/K score · all 8 features",
            "value": 0.0824997209784991
          },
          {
            "label": "Query norm",
            "value": 1.6
          }
        ]
      },
      {
        "id": "reverse",
        "label": "Query 3 · key 1",
        "note": "Reversing the relative direction can change the score even when the distance has the same magnitude.",
        "target": "query_rope",
        "rotation": {
          "queryPosition": 3,
          "keyPosition": 1,
          "query": [
            1,
            0.5
          ],
          "key": [
            0.2,
            1
          ],
          "rotatedQuery": [
            -1.060552500630379,
            -0.35387624024035547
          ],
          "rotatedKey": [
            -0.7334105236342685,
            0.7085965028297191
          ]
        },
        "vectors": [
          {
            "label": "Rotated Q · all 8 features",
            "values": [
              -1.060552500630379,
              -0.35387624024035547,
              0.2797233598235231,
              -0.22749690540541384,
              0.7026845736445407,
              -0.0789581532331518,
              0.19759910360067337,
              0.8005963991027004
            ]
          },
          {
            "label": "Rotated K · all 8 features",
            "values": [
              -0.7334105236342685,
              0.7085965028297191,
              -0.45790171609930724,
              0.5570691325080842,
              0.09699505004141654,
              0.30098498345841623,
              -0.5003997499333541,
              0.39949980008335
            ]
          }
        ],
        "metrics": [
          {
            "label": "Key position − query position",
            "value": -2
          },
          {
            "label": "Scaled Q/K score · all 8 features",
            "value": 0.19006989783022107
          },
          {
            "label": "Query norm",
            "value": 1.6
          }
        ]
      }
    ]
  },
  "revision": "e52c0735ba4f7e05",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/rope/e52c0735ba4f7e05/graph.tensorviz.json",
      "sha256": "89605c8281a07299ba583865e4cd0711b2ae825bfda48b000698cfb648d5c6b3",
      "bytes": 88430
    },
    "model.py": {
      "url": "/models/rope/e52c0735ba4f7e05/model.py",
      "sha256": "a5a791e4b252ae091485fec653a9bd53798407bb6df40d663255744f24a53b7e",
      "bytes": 5369
    },
    "run.py": {
      "url": "/models/rope/e52c0735ba4f7e05/run.py",
      "sha256": "0395a9651d2ae0758cebbb6eefce8fa975b33b242136eb662b6af3639ab68329",
      "bytes": 654
    },
    "requirements.txt": {
      "url": "/models/rope/e52c0735ba4f7e05/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/rope/e52c0735ba4f7e05/preview.svg",
      "sha256": "d8db54ad0a203dca90f212229a2f0ffdaa34f35510f92a3b0659413e120f209f",
      "bytes": 2088
    }
  }
}