{
  "format": "tensorviz.graph",
  "version": 3,
  "exportedAt": "2026-09-17T05:27:48.329770+00:00",
  "status": "static",
  "warningCount": 0,
  "document": {
    "schemaVersion": 2,
    "semantic": {
      "graph_id": "snapshot",
      "name": "GatedFeedForward",
      "revision": 0,
      "nodes": [
        {
          "node_id": "input",
          "kind": "input",
          "op": "Input",
          "params": {
            "shape": "1,4,8"
          },
          "in_ports": [],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "input",
              "name": "Input",
              "displayName": "Input",
              "description": "Model input node",
              "icon": "ArrowRight",
              "category": "io",
              "parameters": [
                {
                  "name": "shape",
                  "type": "string",
                  "defaultValue": "1,3,224,224",
                  "required": true,
                  "description": "Input tensor shape (comma-separated, e.g., 1,3,224,224)"
                },
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "input",
                  "required": false,
                  "description": "Name for this input"
                }
              ]
            }
          }
        },
        {
          "node_id": "plain",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "plain",
            "moduleTemplateId": "model::PlainFeedForward",
            "isGroup": true,
            "moduleMembers": [
              "plain.up",
              "plain.relu",
              "plain.down"
            ],
            "moduleLayerCount": 3
          }
        },
        {
          "node_id": "swiglu",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "swiglu",
            "moduleTemplateId": "model::SwiGLU",
            "isGroup": true,
            "moduleMembers": [
              "swiglu.gate",
              "swiglu.silu",
              "swiglu.up",
              "mul@swiglu#0",
              "swiglu.down"
            ],
            "moduleLayerCount": 5
          }
        },
        {
          "node_id": "stack@GatedFeedForward#0",
          "kind": "layer",
          "op": "torch.stack",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "stack",
              "name": "torch.stack",
              "displayName": "Compare outputs",
              "description": "PyTorch stack operation",
              "icon": "Activity",
              "category": "tensor",
              "parameters": [
                {
                  "name": "dim",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Dimension(s) to apply the operation on"
                }
              ]
            }
          }
        },
        {
          "node_id": "output",
          "kind": "output",
          "op": "Output",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [],
          "metadata": {
            "layerType": {
              "id": "output",
              "name": "Output",
              "displayName": "Output",
              "description": "Model output node",
              "icon": "ArrowLeft",
              "category": "io",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "output",
                  "required": false,
                  "description": "Name for this output"
                }
              ]
            }
          }
        },
        {
          "node_id": "plain.up",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 8,
            "out_features": 24,
            "bias": false
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "plain"
          }
        },
        {
          "node_id": "plain.relu",
          "kind": "layer",
          "op": "ReLU",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "relu",
              "name": "ReLU",
              "displayName": "ReLU",
              "description": "Rectified Linear Unit",
              "icon": "TrendingUp",
              "category": "activation",
              "parameters": [
                {
                  "name": "inplace",
                  "type": "boolean",
                  "defaultValue": false,
                  "required": false,
                  "description": "Can optionally do the operation in-place"
                }
              ]
            },
            "parentId": "plain"
          }
        },
        {
          "node_id": "plain.down",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 24,
            "out_features": 8,
            "bias": false
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "plain"
          }
        },
        {
          "node_id": "swiglu.gate",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 8,
            "out_features": 16,
            "bias": false
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "swiglu"
          }
        },
        {
          "node_id": "swiglu.silu",
          "kind": "layer",
          "op": "SiLU",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "silu",
              "name": "SiLU",
              "displayName": "SiLU",
              "description": "Sigmoid Linear Unit (Swish)",
              "icon": "TrendingUp",
              "category": "activation",
              "parameters": [
                {
                  "name": "inplace",
                  "type": "boolean",
                  "defaultValue": false,
                  "required": false,
                  "description": "Can optionally do the operation in-place"
                }
              ]
            },
            "parentId": "swiglu"
          }
        },
        {
          "node_id": "swiglu.up",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 8,
            "out_features": 16,
            "bias": false
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "swiglu"
          }
        },
        {
          "node_id": "mul@swiglu#0",
          "kind": "layer",
          "op": "torch.mul",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "mul",
              "name": "torch.mul",
              "displayName": "Gate × values",
              "description": "PyTorch mul operation",
              "icon": "Activity",
              "category": "tensor",
              "parameters": [
                {
                  "name": "dim",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Dimension(s) to apply the operation on"
                }
              ]
            },
            "parentId": "swiglu"
          }
        },
        {
          "node_id": "swiglu.down",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 16,
            "out_features": 8,
            "bias": false
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "swiglu"
          }
        }
      ],
      "edges": [
        {
          "edge_id": "edge-1",
          "source_node_id": "plain.up",
          "target_node_id": "plain.relu",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-2",
          "source_node_id": "plain.relu",
          "target_node_id": "plain.down",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-3",
          "source_node_id": "swiglu.gate",
          "target_node_id": "swiglu.silu",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-4",
          "source_node_id": "swiglu.silu",
          "target_node_id": "mul@swiglu#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-5",
          "source_node_id": "swiglu.up",
          "target_node_id": "mul@swiglu#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-6",
          "source_node_id": "mul@swiglu#0",
          "target_node_id": "swiglu.down",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-7",
          "source_node_id": "input",
          "target_node_id": "plain.up",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-8",
          "source_node_id": "input",
          "target_node_id": "plain",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-9",
          "source_node_id": "input",
          "target_node_id": "swiglu.gate",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-10",
          "source_node_id": "input",
          "target_node_id": "swiglu.up",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-11",
          "source_node_id": "input",
          "target_node_id": "swiglu",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-12",
          "source_node_id": "plain.down",
          "target_node_id": "stack@GatedFeedForward#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-13",
          "source_node_id": "plain",
          "target_node_id": "stack@GatedFeedForward#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-14",
          "source_node_id": "swiglu.down",
          "target_node_id": "stack@GatedFeedForward#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-15",
          "source_node_id": "swiglu",
          "target_node_id": "stack@GatedFeedForward#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-16",
          "source_node_id": "stack@GatedFeedForward#0",
          "target_node_id": "output",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        }
      ],
      "inputs": [],
      "modules": [],
      "operations": [],
      "metadata": {}
    },
    "view": {
      "graph_id": "snapshot",
      "node_positions": {
        "input": {
          "x": 0,
          "y": 160
        },
        "plain": {
          "x": 340,
          "y": 160
        },
        "swiglu": {
          "x": 680,
          "y": 160
        },
        "stack@GatedFeedForward#0": {
          "x": 1020,
          "y": 160
        },
        "output": {
          "x": 1360,
          "y": 160
        },
        "plain.up": {
          "x": 24,
          "y": 214
        },
        "plain.relu": {
          "x": 24,
          "y": 364
        },
        "plain.down": {
          "x": 24,
          "y": 514
        },
        "swiglu.gate": {
          "x": 24,
          "y": 214
        },
        "swiglu.silu": {
          "x": 24,
          "y": 364
        },
        "swiglu.up": {
          "x": 24,
          "y": 514
        },
        "mul@swiglu#0": {
          "x": 24,
          "y": 664
        },
        "swiglu.down": {
          "x": 24,
          "y": 814
        }
      },
      "collapsed_modules": [
        "plain",
        "swiglu"
      ],
      "selection": [],
      "layout": {}
    }
  },
  "snapshotId": "snapshot-e635996269cdefa0",
  "collaboration": {
    "walkthrough": {
      "title": "A gate inside the feed-forward block",
      "steps": [
        {
          "id": "plain",
          "title": "A conventional feed-forward route",
          "note": "The ordinary branch projects 8 features to 24, applies ReLU, and projects back to 8. Without biases it has 384 parameters.",
          "view": {
            "camera": {
              "x": 125.5769230769231,
              "y": -86.04195804195803,
              "z": 0.8653555219364599
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "plain": {
                "x": 340,
                "y": 160
              },
              "swiglu": {
                "x": 680,
                "y": 160
              },
              "stack@GatedFeedForward#0": {
                "x": 1020,
                "y": 160
              },
              "output": {
                "x": 1360,
                "y": 160
              },
              "plain.up": {
                "x": 24,
                "y": 214
              },
              "plain.relu": {
                "x": 24,
                "y": 364
              },
              "plain.down": {
                "x": 24,
                "y": 514
              },
              "swiglu.gate": {
                "x": 24,
                "y": 214
              },
              "swiglu.silu": {
                "x": 24,
                "y": 364
              },
              "swiglu.up": {
                "x": 24,
                "y": 514
              },
              "mul@swiglu#0": {
                "x": 24,
                "y": 664
              },
              "swiglu.down": {
                "x": 24,
                "y": 814
              }
            },
            "expandedModuleIds": [
              "plain"
            ],
            "selectedNodeIds": [
              "plain"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "plain"
              },
              "snapshotId": "snapshot-e635996269cdefa0",
              "exportedAt": "2026-09-17T05:27:48.329770+00:00",
              "modelName": "GatedFeedForward",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"plain\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"plain.up\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"plain\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@GatedFeedForward#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"plain.down\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@GatedFeedForward#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"plain.relu\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"plain.down\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"plain.up\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"plain.relu\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"plain.down\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Linear\\\",\\\"params\\\":{\\\"bias\\\":false,\\\"in_features\\\":24,\\\"out_features\\\":8},\\\"parent\\\":\\\"plain\\\"}\",\"{\\\"id\\\":\\\"plain.relu\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"ReLU\\\",\\\"params\\\":{},\\\"parent\\\":\\\"plain\\\"}\",\"{\\\"id\\\":\\\"plain.up\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Linear\\\",\\\"params\\\":{\\\"bias\\\":false,\\\"in_features\\\":8,\\\"out_features\\\":24},\\\"parent\\\":\\\"plain\\\"}\"],\"node\":{\"id\":\"plain\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}"
            }
          ]
        },
        {
          "id": "gate",
          "title": "Make a smooth gate",
          "note": "A learned 8-to-16 projection passes through SiLU, x × sigmoid(x). Unlike a sigmoid probability gate, SiLU can be negative.",
          "view": {
            "camera": {
              "x": 308.5,
              "y": -144,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "plain": {
                "x": 340,
                "y": 160
              },
              "swiglu": {
                "x": 680,
                "y": 160
              },
              "stack@GatedFeedForward#0": {
                "x": 1020,
                "y": 160
              },
              "output": {
                "x": 1360,
                "y": 160
              },
              "plain.up": {
                "x": 24,
                "y": 214
              },
              "plain.relu": {
                "x": 24,
                "y": 364
              },
              "plain.down": {
                "x": 24,
                "y": 514
              },
              "swiglu.gate": {
                "x": 24,
                "y": 214
              },
              "swiglu.silu": {
                "x": 24,
                "y": 364
              },
              "swiglu.up": {
                "x": 24,
                "y": 514
              },
              "mul@swiglu#0": {
                "x": 24,
                "y": 664
              },
              "swiglu.down": {
                "x": 24,
                "y": 814
              }
            },
            "expandedModuleIds": [
              "swiglu"
            ],
            "selectedNodeIds": [
              "swiglu.silu"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "swiglu.silu"
              },
              "snapshotId": "snapshot-e635996269cdefa0",
              "exportedAt": "2026-09-17T05:27:48.329770+00:00",
              "modelName": "GatedFeedForward",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.gate\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"swiglu.silu\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.silu\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@swiglu#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"swiglu.silu\",\"kind\":\"layer\",\"op\":\"SiLU\",\"params\":{},\"parent\":\"swiglu\"}}"
            }
          ]
        },
        {
          "id": "values",
          "title": "Project the values separately",
          "note": "The same input enters another 8-to-16 projection. Multiplying its features by the gate lets one learned branch modulate the other.",
          "view": {
            "camera": {
              "x": 308.5,
              "y": -297.5,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "plain": {
                "x": 340,
                "y": 160
              },
              "swiglu": {
                "x": 680,
                "y": 160
              },
              "stack@GatedFeedForward#0": {
                "x": 1020,
                "y": 160
              },
              "output": {
                "x": 1360,
                "y": 160
              },
              "plain.up": {
                "x": 24,
                "y": 214
              },
              "plain.relu": {
                "x": 24,
                "y": 364
              },
              "plain.down": {
                "x": 24,
                "y": 514
              },
              "swiglu.gate": {
                "x": 24,
                "y": 214
              },
              "swiglu.silu": {
                "x": 24,
                "y": 364
              },
              "swiglu.up": {
                "x": 24,
                "y": 514
              },
              "mul@swiglu#0": {
                "x": 24,
                "y": 664
              },
              "swiglu.down": {
                "x": 24,
                "y": 814
              }
            },
            "expandedModuleIds": [
              "swiglu"
            ],
            "selectedNodeIds": [
              "swiglu.up"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "swiglu.up"
              },
              "snapshotId": "snapshot-e635996269cdefa0",
              "exportedAt": "2026-09-17T05:27:48.329770+00:00",
              "modelName": "GatedFeedForward",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"swiglu.up\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.up\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@swiglu#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"swiglu.up\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":8,\"out_features\":16},\"parent\":\"swiglu\"}}"
            }
          ]
        },
        {
          "id": "output",
          "title": "Return to the model width",
          "note": "The elementwise product has 16 features; the final projection returns 8. Three projections at hidden width 16 also total 384 parameters.",
          "view": {
            "camera": {
              "x": 308.5,
              "y": -597.5,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "plain": {
                "x": 340,
                "y": 160
              },
              "swiglu": {
                "x": 680,
                "y": 160
              },
              "stack@GatedFeedForward#0": {
                "x": 1020,
                "y": 160
              },
              "output": {
                "x": 1360,
                "y": 160
              },
              "plain.up": {
                "x": 24,
                "y": 214
              },
              "plain.relu": {
                "x": 24,
                "y": 364
              },
              "plain.down": {
                "x": 24,
                "y": 514
              },
              "swiglu.gate": {
                "x": 24,
                "y": 214
              },
              "swiglu.silu": {
                "x": 24,
                "y": 364
              },
              "swiglu.up": {
                "x": 24,
                "y": 514
              },
              "mul@swiglu#0": {
                "x": 24,
                "y": 664
              },
              "swiglu.down": {
                "x": 24,
                "y": 814
              }
            },
            "expandedModuleIds": [
              "swiglu"
            ],
            "selectedNodeIds": [
              "swiglu.down"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "swiglu.down"
              },
              "snapshotId": "snapshot-e635996269cdefa0",
              "exportedAt": "2026-09-17T05:27:48.329770+00:00",
              "modelName": "GatedFeedForward",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@swiglu#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"swiglu.down\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.down\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@GatedFeedForward#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"swiglu.down\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":16,\"out_features\":8},\"parent\":\"swiglu\"}}"
            }
          ]
        }
      ]
    },
    "threads": [],
    "savedViews": [
      {
        "id": "gate",
        "title": "Make a smooth gate",
        "note": "A learned 8-to-16 projection passes through SiLU, x × sigmoid(x). Unlike a sigmoid probability gate, SiLU can be negative.",
        "view": {
          "camera": {
            "x": 308.5,
            "y": -144,
            "z": 1.25
          },
          "nodePositions": {
            "input": {
              "x": 0,
              "y": 160
            },
            "plain": {
              "x": 340,
              "y": 160
            },
            "swiglu": {
              "x": 680,
              "y": 160
            },
            "stack@GatedFeedForward#0": {
              "x": 1020,
              "y": 160
            },
            "output": {
              "x": 1360,
              "y": 160
            },
            "plain.up": {
              "x": 24,
              "y": 214
            },
            "plain.relu": {
              "x": 24,
              "y": 364
            },
            "plain.down": {
              "x": 24,
              "y": 514
            },
            "swiglu.gate": {
              "x": 24,
              "y": 214
            },
            "swiglu.silu": {
              "x": 24,
              "y": 364
            },
            "swiglu.up": {
              "x": 24,
              "y": 514
            },
            "mul@swiglu#0": {
              "x": 24,
              "y": 664
            },
            "swiglu.down": {
              "x": 24,
              "y": 814
            }
          },
          "expandedModuleIds": [
            "swiglu"
          ],
          "selectedNodeIds": [
            "swiglu.silu"
          ],
          "viewport": {
            "width": 1100,
            "height": 700
          }
        },
        "anchors": [
          {
            "target": {
              "kind": "node",
              "nodeId": "swiglu.silu"
            },
            "snapshotId": "snapshot-e635996269cdefa0",
            "exportedAt": "2026-09-17T05:27:48.329770+00:00",
            "modelName": "GatedFeedForward",
            "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.gate\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"swiglu.silu\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"swiglu.silu\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@swiglu#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"swiglu.silu\",\"kind\":\"layer\",\"op\":\"SiLU\",\"params\":{},\"parent\":\"swiglu\"}}"
          }
        ]
      }
    ],
    "openingViewId": "gate"
  }
}