{
  "schemaVersion": 1,
  "slug": "swiglu",
  "title": "A gate inside the feed-forward block",
  "className": "GatedFeedForward",
  "question": "Why does this feed-forward block split into two branches?",
  "description": "Trace a SiLU gate, multiply it with a second projection, and compare a conventional FFN at the same parameter count.",
  "category": "Modern LLMs",
  "tags": [
    "SwiGLU",
    "SiLU",
    "Gating",
    "Feed-forward"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "These are untrained feed-forward components. Equal parameter counts do not imply equal quality or runtime. SiLU gate values are not probabilities and can be negative.",
  "openingStep": "gate",
  "provenance": {
    "capturedAt": "2026-09-17T05:27:48.329770+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "610c6468be240a995ad5285135094c1c71ec0204c6ede4db5d72ccd721e09b78",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              4,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      2,
      1,
      4,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One seeded CPU forward plus recorded numerical invariants. Untrained, reduced teaching implementation; no accuracy claim.",
    "numericalChecks": [
      "Both comparison branches contain exactly 384 parameters",
      "Gradients reach the gate, value, and output projections",
      "A zero gate suppresses SwiGLU without changing the ordinary FFN"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,8"
      }
    },
    {
      "id": "plain",
      "label": "plain",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 40,
        "endLine": 40
      }
    },
    {
      "id": "swiglu",
      "label": "swiglu",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 41,
        "endLine": 41
      }
    },
    {
      "id": "stack@GatedFeedForward#0",
      "label": "Compare outputs",
      "op": "torch.stack",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 46,
        "endLine": 46
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "plain.up",
      "label": "plain.up",
      "op": "Linear",
      "kind": "layer",
      "parent": "plain",
      "params": {
        "in_features": 8,
        "out_features": 24,
        "bias": false
      },
      "source": {
        "line": 13,
        "endLine": 13
      }
    },
    {
      "id": "plain.relu",
      "label": "plain.relu",
      "op": "ReLU",
      "kind": "layer",
      "parent": "plain",
      "params": {},
      "source": {
        "line": 14,
        "endLine": 14
      }
    },
    {
      "id": "plain.down",
      "label": "plain.down",
      "op": "Linear",
      "kind": "layer",
      "parent": "plain",
      "params": {
        "in_features": 24,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 15,
        "endLine": 15
      }
    },
    {
      "id": "swiglu.gate",
      "label": "swiglu.gate",
      "op": "Linear",
      "kind": "layer",
      "parent": "swiglu",
      "params": {
        "in_features": 8,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "swiglu.silu",
      "label": "swiglu.silu",
      "op": "SiLU",
      "kind": "layer",
      "parent": "swiglu",
      "params": {},
      "source": {
        "line": 25,
        "endLine": 25
      }
    },
    {
      "id": "swiglu.up",
      "label": "swiglu.up",
      "op": "Linear",
      "kind": "layer",
      "parent": "swiglu",
      "params": {
        "in_features": 8,
        "out_features": 16,
        "bias": false
      },
      "source": {
        "line": 26,
        "endLine": 26
      }
    },
    {
      "id": "mul@swiglu#0",
      "label": "Gate × values",
      "op": "torch.mul",
      "kind": "layer",
      "parent": "swiglu",
      "params": {},
      "source": {
        "line": 32,
        "endLine": 32
      }
    },
    {
      "id": "swiglu.down",
      "label": "swiglu.down",
      "op": "Linear",
      "kind": "layer",
      "parent": "swiglu",
      "params": {
        "in_features": 16,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 27,
        "endLine": 27
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "plain.up",
      "target": "plain.relu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "plain.relu",
      "target": "plain.down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "swiglu.gate",
      "target": "swiglu.silu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "swiglu.silu",
      "target": "mul@swiglu#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "swiglu.up",
      "target": "mul@swiglu#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "mul@swiglu#0",
      "target": "swiglu.down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "input",
      "target": "plain.up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "input",
      "target": "plain",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "input",
      "target": "swiglu.gate",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "input",
      "target": "swiglu.up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "input",
      "target": "swiglu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "plain.down",
      "target": "stack@GatedFeedForward#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "plain",
      "target": "stack@GatedFeedForward#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "swiglu.down",
      "target": "stack@GatedFeedForward#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "swiglu",
      "target": "stack@GatedFeedForward#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "stack@GatedFeedForward#0",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "plain",
      "target": "plain",
      "title": "A conventional feed-forward route",
      "note": "The ordinary branch projects 8 features to 24, applies ReLU, and projects back to 8. Without biases it has 384 parameters."
    },
    {
      "id": "gate",
      "target": "swiglu.silu",
      "title": "Make a smooth gate",
      "note": "A learned 8-to-16 projection passes through SiLU, x × sigmoid(x). Unlike a sigmoid probability gate, SiLU can be negative."
    },
    {
      "id": "values",
      "target": "swiglu.up",
      "title": "Project the values separately",
      "note": "The same input enters another 8-to-16 projection. Multiplying its features by the gate lets one learned branch modulate the other."
    },
    {
      "id": "output",
      "target": "swiglu.down",
      "title": "Return to the model width",
      "note": "The elementwise product has 16 features; the final projection returns 8. Three projections at hidden width 16 also total 384 parameters."
    }
  ],
  "milestone": {
    "name": "SwiGLU",
    "year": 2020,
    "kind": "Component",
    "fidelity": "Bias-free width-8 FFNs: conventional hidden width 24 and SwiGLU hidden width 16, each with 384 parameters.",
    "papers": [
      {
        "title": "GLU Variants Improve Transformer",
        "url": "https://arxiv.org/abs/2002.05202"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "One branch controls another.",
    "description": "Recorded CPU calculations using the same input and seed-0 initialization. The gate is the only changed branch.",
    "controlLabel": "Gate configuration",
    "cases": [
      {
        "id": "learned",
        "label": "Initialized gate",
        "target": "swiglu",
        "note": "Only the gate weights change. SiLU gates can be negative and are not probabilities. These initialized weights illustrate arithmetic, not learned language behavior.",
        "vectors": [
          {
            "label": "SiLU gate · 16 features",
            "values": [
              0.43659564234302,
              -0.23519764726481218,
              0.4880266811347049,
              -0.2782562805784101,
              0.3056657505398149,
              -0.2343494737839157,
              0.6628327222544355,
              0.5859662301570313,
              -0.12314768192564843,
              0.3373330846063079,
              -0.2757964109044825,
              -0.2784391444951643,
              0.2843640811959047,
              0.555528012281415,
              1.0509316488648317,
              0.27128436989341065
            ]
          },
          {
            "label": "Value branch · 16 features",
            "values": [
              1.282099511474371,
              0.6700474284589291,
              1.218139011238236,
              1.0658179838210344,
              -0.04006325639784336,
              1.1914814542979002,
              0.8918015249073505,
              -0.19675150141119957,
              0.7623524218797684,
              1.3856845162808895,
              -0.3392070159316063,
              1.8240600042045116,
              -1.96096402220428,
              1.8205962926149368,
              0.8460588329471648,
              -1.0807585939764977
            ]
          },
          {
            "label": "Elementwise product · 16 features",
            "values": [
              0.5597590597598251,
              -0.15759357872937768,
              0.5944843388153073,
              -0.29657054795162113,
              -0.012245965335915831,
              -0.27922305183800755,
              0.5911152324649959,
              -0.11528973555965644,
              -0.09388193356489746,
              0.46743723216823213,
              0.09355207754755662,
              -0.5078897070785501,
              -0.5576277324323458,
              1.0113922396032893,
              0.889150004345819,
              -0.2931929141738026
            ]
          },
          {
            "label": "Output projection · 8 features",
            "values": [
              -0.6103001573487269,
              0.08338835631230446,
              0.11831706100967253,
              -0.04377586976983599,
              -0.058996601643856866,
              -0.03362481026480574,
              0.15888628798669205,
              -0.43191659990002845
            ]
          }
        ],
        "metrics": [
          {
            "label": "Ordinary FFN parameters",
            "value": 384
          },
          {
            "label": "SwiGLU parameters",
            "value": 384
          }
        ]
      },
      {
        "id": "zero",
        "label": "Zero gate weights",
        "target": "swiglu",
        "note": "Only the gate weights change. SiLU gates can be negative and are not probabilities. These initialized weights illustrate arithmetic, not learned language behavior.",
        "vectors": [
          {
            "label": "SiLU gate · 16 features",
            "values": [
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Value branch · 16 features",
            "values": [
              1.282099511474371,
              0.6700474284589291,
              1.218139011238236,
              1.0658179838210344,
              -0.04006325639784336,
              1.1914814542979002,
              0.8918015249073505,
              -0.19675150141119957,
              0.7623524218797684,
              1.3856845162808895,
              -0.3392070159316063,
              1.8240600042045116,
              -1.96096402220428,
              1.8205962926149368,
              0.8460588329471648,
              -1.0807585939764977
            ]
          },
          {
            "label": "Elementwise product · 16 features",
            "values": [
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Output projection · 8 features",
            "values": [
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          }
        ],
        "metrics": [
          {
            "label": "Ordinary FFN parameters",
            "value": 384
          },
          {
            "label": "SwiGLU parameters",
            "value": 384
          }
        ]
      },
      {
        "id": "double",
        "label": "Double gate weights",
        "target": "swiglu",
        "note": "Only the gate weights change. SiLU gates can be negative and are not probabilities. These initialized weights illustrate arithmetic, not learned language behavior.",
        "vectors": [
          {
            "label": "SiLU gate · 16 features",
            "values": [
              1.045426786290373,
              -0.27601696375053164,
              1.1734783673851887,
              -0.19260002760192743,
              0.7171109665465752,
              -0.27630968298669456,
              1.5997770464824128,
              1.4142671801578872,
              -0.20694729259499972,
              0.7965583322431034,
              -0.1529070874404189,
              -0.18701547950123143,
              0.663782463761934,
              1.339926464764254,
              2.484177243786751,
              0.6311099285136484
            ]
          },
          {
            "label": "Value branch · 16 features",
            "values": [
              1.282099511474371,
              0.6700474284589291,
              1.218139011238236,
              1.0658179838210344,
              -0.04006325639784336,
              1.1914814542979002,
              0.8918015249073505,
              -0.19675150141119957,
              0.7623524218797684,
              1.3856845162808895,
              -0.3392070159316063,
              1.8240600042045116,
              -1.96096402220428,
              1.8205962926149368,
              0.8460588329471648,
              -1.0807585939764977
            ]
          },
          {
            "label": "Elementwise product · 16 features",
            "values": [
              1.3403411719851088,
              -0.18494445677208515,
              1.4294597781560532,
              -0.20527657310256187,
              -0.02872980051846072,
              -0.3292178629215786,
              1.4266836095647932,
              -0.2782591910926478,
              -0.15776676971125908,
              1.1037785473037967,
              0.05186715684545769,
              -0.341127456325325,
              -1.3016535300072687,
              2.4394651541264394,
              2.101760099712123,
              -0.6820774789850186
            ]
          },
          {
            "label": "Output projection · 8 features",
            "values": [
              -1.4837895515415078,
              0.4532055856607805,
              0.4089782231987111,
              -0.04508961391767695,
              -0.09094572217442112,
              0.033744322780925484,
              0.12784889469335192,
              -0.6591696353370882
            ]
          }
        ],
        "metrics": [
          {
            "label": "Ordinary FFN parameters",
            "value": 384
          },
          {
            "label": "SwiGLU parameters",
            "value": 384
          }
        ]
      }
    ]
  },
  "revision": "2e269c77c5e058b6",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/swiglu/2e269c77c5e058b6/graph.tensorviz.json",
      "sha256": "cd8bd0f886e62660d6c4d355df67dcbc9d3e6fae031fe4c7582f33f382e29c92",
      "bytes": 36381
    },
    "model.py": {
      "url": "/models/swiglu/2e269c77c5e058b6/model.py",
      "sha256": "610c6468be240a995ad5285135094c1c71ec0204c6ede4db5d72ccd721e09b78",
      "bytes": 3808
    },
    "run.py": {
      "url": "/models/swiglu/2e269c77c5e058b6/run.py",
      "sha256": "e3c8aaa894ce3af1935f56102711f56c6ccf0307096fc7ef7fcb20e9e3de5dd0",
      "bytes": 658
    },
    "requirements.txt": {
      "url": "/models/swiglu/2e269c77c5e058b6/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/swiglu/2e269c77c5e058b6/preview.svg",
      "sha256": "2b2fc344038dcb85d60838daea7c1f5cb37cb9c56a175530bf7dcd9942d3c769",
      "bytes": 2250
    }
  }
}