{
  "schemaVersion": 1,
  "slug": "pre-norm",
  "title": "Where the normalization goes",
  "className": "NormPlacement",
  "question": "What happens to the shortcut when normalization moves?",
  "description": "Keep the learned branch fixed while comparing pre-norm and post-norm, then zero the branch to expose the identity path.",
  "category": "Modern LLMs",
  "tags": [
    "Pre-norm",
    "Post-norm",
    "LayerNorm",
    "Residual"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "Both branches use LayerNorm. This comparison changes placement, not RMSNorm versus LayerNorm. Forward values and a local derivative do not establish training stability for a full network.",
  "openingStep": "shortcut",
  "provenance": {
    "capturedAt": "2026-09-17T05:43:03.909352+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "cb5e47f19519d2dba172bb95821ca71583e7123cd9c3f3b92618ee3a58436a62",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              4,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      2,
      1,
      4,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One seeded CPU forward plus identical-weight, zero-branch and local-derivative checks. No optimization or training-speed benchmark.",
    "numericalChecks": [
      "Both placements use identical branch weights",
      "Zeroing F leaves pre-norm's identity path intact but post-norm still normalizes",
      "The zero-branch pre-norm path has the identity derivative"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,8"
      }
    },
    {
      "id": "pre",
      "label": "pre",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 38,
        "endLine": 38
      }
    },
    {
      "id": "post",
      "label": "post",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 39,
        "endLine": 39
      }
    },
    {
      "id": "stack@NormPlacement#0",
      "label": "Compare placements",
      "op": "torch.stack",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 45,
        "endLine": 45
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "pre.norm",
      "label": "pre.norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "pre",
      "params": {
        "normalized_shape": 8,
        "eps": 0.000001
      },
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "pre.branch",
      "label": "pre.branch",
      "op": "Linear",
      "kind": "layer",
      "parent": "pre",
      "params": {
        "in_features": 8,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 14,
        "endLine": 14
      }
    },
    {
      "id": "add@pre#0",
      "label": "Pre-norm residual",
      "op": "torch.add",
      "kind": "layer",
      "parent": "pre",
      "params": {},
      "source": {
        "line": 19,
        "endLine": 19
      }
    },
    {
      "id": "post.branch",
      "label": "post.branch",
      "op": "Linear",
      "kind": "layer",
      "parent": "post",
      "params": {
        "in_features": 8,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 25,
        "endLine": 25
      }
    },
    {
      "id": "add@post#0",
      "label": "Post-norm residual",
      "op": "torch.add",
      "kind": "layer",
      "parent": "post",
      "params": {},
      "source": {
        "line": 30,
        "endLine": 30
      }
    },
    {
      "id": "post.norm",
      "label": "post.norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "post",
      "params": {
        "normalized_shape": 8,
        "eps": 0.000001
      },
      "source": {
        "line": 26,
        "endLine": 26
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "pre.norm",
      "target": "pre.branch",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "pre.branch",
      "target": "add@pre#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "post.branch",
      "target": "add@post#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "add@post#0",
      "target": "post.norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "input",
      "target": "pre.norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "input",
      "target": "add@pre#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "input",
      "target": "pre",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "input",
      "target": "post.branch",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "input",
      "target": "add@post#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "input",
      "target": "post",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "add@pre#0",
      "target": "stack@NormPlacement#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "pre",
      "target": "stack@NormPlacement#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "post.norm",
      "target": "stack@NormPlacement#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "post",
      "target": "stack@NormPlacement#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "stack@NormPlacement#0",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "pre",
      "target": "pre.norm",
      "title": "Normalize inside the branch",
      "note": "Pre-norm applies LayerNorm before F. The original input travels around both operations, so the result is x + F(Norm(x))."
    },
    {
      "id": "shortcut",
      "target": "add@pre#0",
      "title": "Keep the identity path intact",
      "note": "At the addition, the original x rejoins the transformed branch. Set F to zero in the recorded comparison: this output becomes exactly x."
    },
    {
      "id": "post-branch",
      "target": "post.branch",
      "title": "Use the same F before normalization",
      "note": "The post-norm branch starts with the same F weights, but F receives the raw input. Its output is added to x before normalization."
    },
    {
      "id": "post",
      "target": "post.norm",
      "title": "Normalize after the addition",
      "note": "Post-norm computes Norm(x + F(x)). Even with F set to zero, the output is Norm(x), so normalization remains on the residual route."
    }
  ],
  "milestone": {
    "name": "Pre-LN / Post-LN",
    "year": 2020,
    "kind": "Component",
    "fidelity": "An eight-feature residual sublayer with a linear F isolates normalization placement. The cited 2020 paper analyzes the already proposed pre-LN design.",
    "papers": [
      {
        "title": "On Layer Normalization in the Transformer Architecture",
        "url": "https://arxiv.org/abs/2002.04745"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "Move the normalization. Keep F fixed.",
    "description": "Recorded CPU calculations with the same eight-feature input, matching branch weights, and matching LayerNorm parameters.",
    "controlLabel": "Residual branch",
    "cases": [
      {
        "id": "same-weights",
        "label": "Same initialized F",
        "note": "Both branches have the same F weights and LayerNorm parameters. Only the placement changes: x + F(Norm(x)) versus Norm(x + F(x)).",
        "target": "pre",
        "vectors": [
          {
            "label": "Input",
            "values": [
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8
            ]
          },
          {
            "label": "Pre-norm output",
            "values": [
              1.4972997542723085,
              1.6649569068645702,
              3.2905059572215976,
              4.117303462530286,
              6.3494025138431365,
              6.587216715444726,
              7.03124934128987,
              8.461359463920441
            ]
          },
          {
            "label": "Post-norm output",
            "values": [
              -0.8700548179709824,
              -2.007308416829704,
              0.4661034948453227,
              -0.31488725653996547,
              0.2972447652097849,
              0.21683505826462385,
              1.5030965232616023,
              0.7089706497593183
            ]
          }
        ],
        "metrics": [
          {
            "label": "Pre-norm distance from input",
            "value": 1.6844151581198032
          },
          {
            "label": "Post-norm distance from input",
            "value": 13.547341793341246
          }
        ]
      },
      {
        "id": "zero-branch",
        "label": "Set F to zero",
        "note": "The pre-norm residual is exactly x. Post-norm still transforms x through LayerNorm. This isolates the identity path without making a training-stability claim.",
        "target": "pre",
        "vectors": [
          {
            "label": "Input",
            "values": [
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8
            ]
          },
          {
            "label": "Pre-norm output",
            "values": [
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8
            ]
          },
          {
            "label": "Post-norm output",
            "values": [
              -1.527525086173374,
              -1.0910893472666958,
              -0.6546536083600174,
              -0.21821786945333915,
              0.21821786945333915,
              0.6546536083600174,
              1.0910893472666958,
              1.527525086173374
            ]
          }
        ],
        "metrics": [
          {
            "label": "Pre-norm distance from input",
            "value": 0
          },
          {
            "label": "Post-norm distance from input",
            "value": 13.241578320125958
          }
        ]
      }
    ]
  },
  "revision": "498438923bc427aa",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/pre-norm/498438923bc427aa/graph.tensorviz.json",
      "sha256": "f20c2bdfd41ce433f59d0ad2295dc1237e14682db2fb72196c194abc695005b0",
      "bytes": 31577
    },
    "model.py": {
      "url": "/models/pre-norm/498438923bc427aa/model.py",
      "sha256": "cb5e47f19519d2dba172bb95821ca71583e7123cd9c3f3b92618ee3a58436a62",
      "bytes": 3630
    },
    "run.py": {
      "url": "/models/pre-norm/498438923bc427aa/run.py",
      "sha256": "5d99f4db8693de5126eb454a10a5a84ca710fc9f4007878e8f69312507bf3814",
      "bytes": 652
    },
    "requirements.txt": {
      "url": "/models/pre-norm/498438923bc427aa/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/pre-norm/498438923bc427aa/preview.svg",
      "sha256": "fcbb46167757fc5665fb7925ccfcd0c4546cd2112b464d0590c274ffce182f8d",
      "bytes": 2171
    }
  }
}