{ "format": "tensorviz.graph", "version": 3, "exportedAt": "2026-09-17T05:27:44.577872+00:00", "status": "static", "warningCount": 0, "document": { "schemaVersion": 2, "semantic": { "graph_id": "snapshot", "name": "NormalizationComparison", "revision": 0, "nodes": [ { "node_id": "input", "kind": "input", "op": "Input", "params": { "shape": "1,4,8" }, "in_ports": [], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "input", "name": "Input", "displayName": "Input", "description": "Model input node", "icon": "ArrowRight", "category": "io", "parameters": [ { "name": "shape", "type": "string", "defaultValue": "1,3,224,224", "required": true, "description": "Input tensor shape (comma-separated, e.g., 1,3,224,224)" }, { "name": "name", "type": "string", "defaultValue": "input", "required": false, "description": "Name for this input" } ] } } }, { "node_id": "layer_norm", "kind": "layer", "op": "LayerNorm", "params": { "normalized_shape": 8, "eps": 0.000001 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "layernorm", "name": "LayerNorm", "displayName": "LayerNorm", "description": "Layer Normalization", "icon": "Layers", "category": "norm", "parameters": [ { "name": "normalized_shape", "type": "string", "defaultValue": "512", "required": true, "description": "Input shape for normalization (comma-separated for multi-dim, e.g., 512)" }, { "name": "eps", "type": "number", "defaultValue": 0.00001, "required": false, "description": "Value for numerical stability", "min": 0 }, { "name": "elementwise_affine", "type": "boolean", "defaultValue": true, "required": false, "description": "If True, learnable affine parameters" } ] } } }, { "node_id": "rms_norm", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "moduleInstanceId": "rms_norm", "moduleTemplateId": "model::RMSNorm", "isGroup": true, "moduleMembers": [ "rms_norm.rms", "div@rms_norm#0", "rms_norm.gain", "mul@rms_norm#0" ], "moduleLayerCount": 4 } }, { "node_id": "stack@NormalizationComparison#0", "kind": "layer", "op": "torch.stack", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "stack", "name": "torch.stack", "displayName": "Compare outputs", "description": "PyTorch stack operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] } } }, { "node_id": "output", "kind": "output", "op": "Output", "params": {}, "in_ports": [ "x" ], "out_ports": [], "metadata": { "layerType": { "id": "output", "name": "Output", "displayName": "Output", "description": "Model output node", "icon": "ArrowLeft", "category": "io", "parameters": [ { "name": "name", "type": "string", "defaultValue": "output", "required": false, "description": "Name for this output" } ] } } }, { "node_id": "rms_norm.rms", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "rms_norm", "moduleInstanceId": "rms_norm.rms", "moduleTemplateId": "model::RootMeanSquare", "isGroup": true, "moduleMembers": [ "python@rms_norm.rms#0" ], "moduleLayerCount": 1 } }, { "node_id": "div@rms_norm#0", "kind": "layer", "op": "torch.div", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "div", "name": "torch.div", "displayName": "Divide by RMS", "description": "PyTorch div operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "rms_norm" } }, { "node_id": "rms_norm.gain", "kind": "layer", "op": "Parameter", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "parameter", "description": "", "category": "misc", "icon": "Box", "parameters": [], "name": "Parameter", "displayName": "Parameter" }, "parentId": "rms_norm" } }, { "node_id": "mul@rms_norm#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "Apply learned gain", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "rms_norm" } }, { "node_id": "python@rms_norm.rms#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Root mean square ยท Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "rms_norm.rms" } } ], "edges": [ { "edge_id": "edge-1", "source_node_id": "python@rms_norm.rms#0", "target_node_id": "div@rms_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-2", "source_node_id": "rms_norm.rms", "target_node_id": "div@rms_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-3", "source_node_id": "div@rms_norm#0", "target_node_id": "mul@rms_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-4", "source_node_id": "rms_norm.gain", "target_node_id": "mul@rms_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-5", "source_node_id": "input", "target_node_id": "layer_norm", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-6", "source_node_id": "input", "target_node_id": "python@rms_norm.rms#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-7", "source_node_id": "input", "target_node_id": "div@rms_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-8", "source_node_id": "input", "target_node_id": "rms_norm", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-9", "source_node_id": "layer_norm", "target_node_id": "stack@NormalizationComparison#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-10", "source_node_id": "mul@rms_norm#0", "target_node_id": "stack@NormalizationComparison#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-11", "source_node_id": "rms_norm", "target_node_id": "stack@NormalizationComparison#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-12", "source_node_id": "stack@NormalizationComparison#0", "target_node_id": "output", "source_port": "y", "target_port": "x", "metadata": {} } ], "inputs": [], "modules": [], "operations": [], "metadata": {} }, "view": { "graph_id": "snapshot", "node_positions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "collapsed_modules": [ "rms_norm", "rms_norm.rms" ], "selection": [], "layout": {} } }, "snapshotId": "snapshot-86c03be49bb2aeb2", "collaboration": { "walkthrough": { "title": "Scale without centering", "steps": [ { "id": "input", "title": "Start with the same features", "note": "Both normalization branches receive the same [1, 4, 8] input. Normalization acts along each token's eight-feature axis.", "view": { "camera": { "x": 332.5, "y": 60, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "expandedModuleIds": [], "selectedNodeIds": [ "input" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "input" }, "snapshotId": "snapshot-86c03be49bb2aeb2", "exportedAt": "2026-09-17T05:27:44.577872+00:00", "modelName": "NormalizationComparison", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"div@rms_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"layer_norm\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@rms_norm.rms#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"rms_norm\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"input\",\"kind\":\"input\",\"op\":\"Input\",\"params\":{\"shape\":\"1,4,8\"}}}" } ] }, { "id": "center", "title": "Subtract the mean in LayerNorm", "note": "LayerNorm subtracts each token's feature mean and divides by the square root of variance plus epsilon. Its learned gain and bias start at one and zero.", "view": { "camera": { "x": -7.5, "y": 60, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "expandedModuleIds": [], "selectedNodeIds": [ "layer_norm" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "layer_norm" }, "snapshotId": "snapshot-86c03be49bb2aeb2", "exportedAt": "2026-09-17T05:27:44.577872+00:00", "modelName": "NormalizationComparison", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"layer_norm\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"layer_norm\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@NormalizationComparison#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"layer_norm\",\"kind\":\"layer\",\"op\":\"LayerNorm\",\"params\":{\"eps\":0.000001,\"normalized_shape\":8}}}" } ] }, { "id": "rms", "title": "Measure root mean square", "note": "RMSNorm uses the mean of squared features plus epsilon, then a square root. It does not subtract the feature mean. TensorViz groups these tensor expressions in this Python node.", "view": { "camera": { "x": 308.5, "y": 6, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "expandedModuleIds": [ "rms_norm", "rms_norm.rms" ], "selectedNodeIds": [ "python@rms_norm.rms#0" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "python@rms_norm.rms#0" }, "snapshotId": "snapshot-86c03be49bb2aeb2", "exportedAt": "2026-09-17T05:27:44.577872+00:00", "modelName": "NormalizationComparison", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@rms_norm.rms#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@rms_norm.rms#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"div@rms_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"python@rms_norm.rms#0\",\"kind\":\"layer\",\"op\":\"Python\",\"params\":{},\"parent\":\"rms_norm.rms\"}}" } ] }, { "id": "gain", "title": "Rescale and apply a learned gain", "note": "Dividing by RMS preserves the effect of an input offset. A learned eight-feature gain rescales the result. Compare the recorded vectors below.", "view": { "camera": { "x": 308.5, "y": -444, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "expandedModuleIds": [ "rms_norm" ], "selectedNodeIds": [ "mul@rms_norm#0" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "mul@rms_norm#0" }, "snapshotId": "snapshot-86c03be49bb2aeb2", "exportedAt": "2026-09-17T05:27:44.577872+00:00", "modelName": "NormalizationComparison", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"div@rms_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@rms_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@rms_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@NormalizationComparison#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"rms_norm.gain\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@rms_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"mul@rms_norm#0\",\"kind\":\"layer\",\"op\":\"torch.mul\",\"params\":{},\"parent\":\"rms_norm\"}}" } ] } ] }, "threads": [], "savedViews": [ { "id": "rms", "title": "Measure root mean square", "note": "RMSNorm uses the mean of squared features plus epsilon, then a square root. It does not subtract the feature mean. TensorViz groups these tensor expressions in this Python node.", "view": { "camera": { "x": 308.5, "y": 6, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "layer_norm": { "x": 340, "y": 160 }, "rms_norm": { "x": 680, "y": 160 }, "stack@NormalizationComparison#0": { "x": 1020, "y": 160 }, "output": { "x": 1360, "y": 160 }, "rms_norm.rms": { "x": 24, "y": 214 }, "div@rms_norm#0": { "x": 24, "y": 364 }, "rms_norm.gain": { "x": 24, "y": 514 }, "mul@rms_norm#0": { "x": 24, "y": 664 }, "python@rms_norm.rms#0": { "x": 24, "y": 214 } }, "expandedModuleIds": [ "rms_norm", "rms_norm.rms" ], "selectedNodeIds": [ "python@rms_norm.rms#0" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "python@rms_norm.rms#0" }, "snapshotId": "snapshot-86c03be49bb2aeb2", "exportedAt": "2026-09-17T05:27:44.577872+00:00", "modelName": "NormalizationComparison", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@rms_norm.rms#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@rms_norm.rms#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"div@rms_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"python@rms_norm.rms#0\",\"kind\":\"layer\",\"op\":\"Python\",\"params\":{},\"parent\":\"rms_norm.rms\"}}" } ] } ], "openingViewId": "rms" } }