{
  "schemaVersion": 1,
  "slug": "rmsnorm",
  "title": "Scale without centering",
  "className": "NormalizationComparison",
  "question": "What does normalization change in one token vector?",
  "description": "Compare LayerNorm and RMSNorm on the same eight features, then change the input's offset and scale.",
  "category": "Modern LLMs",
  "tags": [
    "RMSNorm",
    "LayerNorm",
    "Normalization"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "Normalization type and placement are separate choices. This forward comparison does not establish training stability or speed. The recorded experiment uses float64; the captured graph input uses float32.",
  "openingStep": "rms",
  "provenance": {
    "capturedAt": "2026-09-17T05:27:44.577872+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "cb7f385265455049f0b476eb05031c6efb964dec3f92b8420bb1d4a32ddef028",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              4,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      2,
      1,
      4,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One seeded CPU forward plus recorded numerical invariants. Untrained, reduced teaching implementation; no accuracy claim.",
    "numericalChecks": [
      "LayerNorm removes a common offset; RMSNorm preserves its effect",
      "RMS normalization and positive rescaling agree within epsilon tolerance",
      "Zero input stays finite and learned gain rescales the result"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,8"
      }
    },
    {
      "id": "layer_norm",
      "label": "layer_norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": null,
      "params": {
        "normalized_shape": 8,
        "eps": 0.000001
      },
      "source": {
        "line": 29,
        "endLine": 29
      }
    },
    {
      "id": "rms_norm",
      "label": "rms_norm",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 30,
        "endLine": 30
      }
    },
    {
      "id": "stack@NormalizationComparison#0",
      "label": "Compare outputs",
      "op": "torch.stack",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "rms_norm.rms",
      "label": "rms_norm.rms",
      "op": "Module",
      "kind": "module",
      "parent": "rms_norm",
      "params": {},
      "source": {
        "line": 17,
        "endLine": 17
      }
    },
    {
      "id": "div@rms_norm#0",
      "label": "Divide by RMS",
      "op": "torch.div",
      "kind": "layer",
      "parent": "rms_norm",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "rms_norm.gain",
      "label": "rms_norm.gain",
      "op": "Parameter",
      "kind": "layer",
      "parent": "rms_norm",
      "params": {},
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "mul@rms_norm#0",
      "label": "Apply learned gain",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "rms_norm",
      "params": {},
      "source": {
        "line": 22,
        "endLine": 22
      }
    },
    {
      "id": "python@rms_norm.rms#0",
      "label": "Root mean square · Python",
      "op": "Python",
      "kind": "layer",
      "parent": "rms_norm.rms",
      "params": {},
      "source": {
        "line": 11,
        "endLine": 11
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "python@rms_norm.rms#0",
      "target": "div@rms_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "rms_norm.rms",
      "target": "div@rms_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "div@rms_norm#0",
      "target": "mul@rms_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "rms_norm.gain",
      "target": "mul@rms_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "input",
      "target": "layer_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "input",
      "target": "python@rms_norm.rms#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "input",
      "target": "div@rms_norm#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "input",
      "target": "rms_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "layer_norm",
      "target": "stack@NormalizationComparison#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "mul@rms_norm#0",
      "target": "stack@NormalizationComparison#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "rms_norm",
      "target": "stack@NormalizationComparison#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "stack@NormalizationComparison#0",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "input",
      "target": "input",
      "title": "Start with the same features",
      "note": "Both normalization branches receive the same [1, 4, 8] input. Normalization acts along each token's eight-feature axis."
    },
    {
      "id": "center",
      "target": "layer_norm",
      "title": "Subtract the mean in LayerNorm",
      "note": "LayerNorm subtracts each token's feature mean and divides by the square root of variance plus epsilon. Its learned gain and bias start at one and zero."
    },
    {
      "id": "rms",
      "target": "python@rms_norm.rms#0",
      "title": "Measure root mean square",
      "note": "RMSNorm uses the mean of squared features plus epsilon, then a square root. It does not subtract the feature mean. TensorViz groups these tensor expressions in this Python node."
    },
    {
      "id": "gain",
      "target": "mul@rms_norm#0",
      "title": "Rescale and apply a learned gain",
      "note": "Dividing by RMS preserves the effect of an input offset. A learned eight-feature gain rescales the result. Compare the recorded vectors below."
    }
  ],
  "milestone": {
    "name": "RMSNorm",
    "year": 2019,
    "kind": "Component",
    "fidelity": "Eight-feature implementation of the normalization equations; unit gains at initialization.",
    "papers": [
      {
        "title": "Root Mean Square Layer Normalization",
        "url": "https://arxiv.org/abs/1910.07467"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "Keep the vector. Change the normalization.",
    "description": "Recorded CPU calculations with unit gains, zero LayerNorm bias, and epsilon 0.000001. Choose an input transformation.",
    "controlLabel": "Input transformation",
    "cases": [
      {
        "id": "balanced",
        "label": "Balanced vector",
        "note": "Both formulas agree for this zero-mean vector when their gains and epsilon match.",
        "target": "rms_norm",
        "vectors": [
          {
            "label": "Input · 8 features",
            "values": [
              1,
              2,
              3,
              4,
              -1,
              -2,
              -3,
              -4
            ]
          },
          {
            "label": "LayerNorm",
            "values": [
              0.3651483473268884,
              0.7302966946537768,
              1.0954450419806652,
              1.4605933893075536,
              -0.3651483473268884,
              -0.7302966946537768,
              -1.0954450419806652,
              -1.4605933893075536
            ]
          },
          {
            "label": "RMSNorm",
            "values": [
              0.3651483473268884,
              0.7302966946537768,
              1.0954450419806652,
              1.4605933893075536,
              -0.3651483473268884,
              -0.7302966946537768,
              -1.0954450419806652,
              -1.4605933893075536
            ]
          }
        ],
        "metrics": [
          {
            "label": "LayerNorm output mean",
            "value": 0
          },
          {
            "label": "RMSNorm output mean",
            "value": 0
          }
        ]
      },
      {
        "id": "offset",
        "label": "Add 5 to every feature",
        "note": "LayerNorm removes the common offset. RMSNorm retains a positive mean while rescaling the vector.",
        "target": "rms_norm",
        "vectors": [
          {
            "label": "Input · 8 features",
            "values": [
              6,
              7,
              8,
              9,
              4,
              3,
              2,
              1
            ]
          },
          {
            "label": "LayerNorm",
            "values": [
              0.3651483473268884,
              0.7302966946537768,
              1.0954450419806652,
              1.4605933893075536,
              -0.3651483473268884,
              -0.7302966946537768,
              -1.0954450419806652,
              -1.4605933893075536
            ]
          },
          {
            "label": "RMSNorm",
            "values": [
              1.052469606976595,
              1.2278812081393609,
              1.4032928093021266,
              1.5787044104648926,
              0.7016464046510633,
              0.5262348034882975,
              0.35082320232553166,
              0.17541160116276583
            ]
          }
        ],
        "metrics": [
          {
            "label": "LayerNorm output mean",
            "value": 0
          },
          {
            "label": "RMSNorm output mean",
            "value": 0.8770580058138291
          }
        ]
      },
      {
        "id": "scaled",
        "label": "Multiply every feature by 10",
        "note": "Both outputs nearly match the balanced case; epsilon introduces a small numerical difference.",
        "target": "rms_norm",
        "vectors": [
          {
            "label": "Input · 8 features",
            "values": [
              10,
              20,
              30,
              40,
              -10,
              -20,
              -30,
              -40
            ]
          },
          {
            "label": "LayerNorm",
            "values": [
              0.3651483714266785,
              0.730296742853357,
              1.0954451142800354,
              1.460593485706714,
              -0.3651483714266785,
              -0.730296742853357,
              -1.0954451142800354,
              -1.460593485706714
            ]
          },
          {
            "label": "RMSNorm",
            "values": [
              0.3651483714266785,
              0.730296742853357,
              1.0954451142800354,
              1.460593485706714,
              -0.3651483714266785,
              -0.730296742853357,
              -1.0954451142800354,
              -1.460593485706714
            ]
          }
        ],
        "metrics": [
          {
            "label": "LayerNorm output mean",
            "value": -2.7755575615628914e-17
          },
          {
            "label": "RMSNorm output mean",
            "value": -2.7755575615628914e-17
          }
        ]
      }
    ]
  },
  "revision": "57dce70f3f8f53cb",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/rmsnorm/57dce70f3f8f53cb/graph.tensorviz.json",
      "sha256": "eeeaf6a8fc2567991247d05f01af9f882a4abe6b35af9e47c88b7faec2e55484",
      "bytes": 27921
    },
    "model.py": {
      "url": "/models/rmsnorm/57dce70f3f8f53cb/model.py",
      "sha256": "cb7f385265455049f0b476eb05031c6efb964dec3f92b8420bb1d4a32ddef028",
      "bytes": 3346
    },
    "run.py": {
      "url": "/models/rmsnorm/57dce70f3f8f53cb/run.py",
      "sha256": "b34012fd5744793cfb448ffeab238519baea5e04e26f0656581800705dbc2897",
      "bytes": 672
    },
    "requirements.txt": {
      "url": "/models/rmsnorm/57dce70f3f8f53cb/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/rmsnorm/57dce70f3f8f53cb/preview.svg",
      "sha256": "af127c924d2f4ac6b1323cfa6e2c9fec24947f1fe6a2e0ab7b07b32655216c26",
      "bytes": 1571
    }
  }
}