{ "format": "tensorviz.graph", "version": 3, "exportedAt": "2026-09-17T06:13:04.092484+00:00", "status": "static", "warningCount": 0, "document": { "schemaVersion": 2, "semantic": { "graph_id": "snapshot", "name": "TinyLlama", "revision": 0, "nodes": [ { "node_id": "input", "kind": "input", "op": "Input", "params": { "shape": "1,4,6,4" }, "in_ports": [], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "input", "name": "Input", "displayName": "Input", "description": "Model input node", "icon": "ArrowRight", "category": "io", "parameters": [ { "name": "shape", "type": "string", "defaultValue": "1,3,224,224", "required": true, "description": "Input tensor shape (comma-separated, e.g., 1,3,224,224)" }, { "name": "name", "type": "string", "defaultValue": "input", "required": false, "description": "Name for this input" } ] } } }, { "node_id": "embedding", "kind": "layer", "op": "Embedding", "params": { "num_embeddings": 32, "embedding_dim": 16 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "embedding", "name": "Embedding", "displayName": "Embedding", "description": "Lookup table for dense vector embeddings", "icon": "Hash", "category": "embedding", "parameters": [ { "name": "num_embeddings", "type": "number", "defaultValue": 10000, "required": true, "description": "Size of the dictionary of embeddings", "min": 1 }, { "name": "embedding_dim", "type": "number", "defaultValue": 128, "required": true, "description": "Size of each embedding vector", "min": 1 }, { "name": "padding_idx", "type": "number", "defaultValue": 0, "required": false, "description": "Pad output with zeros at this index", "min": 0 }, { "name": "max_norm", "type": "number", "defaultValue": 0, "required": false, "description": "If > 0, re-normalizes embeddings with norm > max_norm", "min": 0 }, { "name": "sparse", "type": "boolean", "defaultValue": false, "required": false, "description": "If True, gradient w.r.t. weight is sparse" } ] } } }, { "node_id": "block", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "moduleInstanceId": "block", "moduleTemplateId": "model::DecoderBlock", "isGroup": true, "moduleMembers": [ "block.attention_norm", "block.attention", "add@block#0", "block.ffn_norm", "block.ffn", "add@block#1" ], "moduleLayerCount": 43 } }, { "node_id": "final_norm", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "moduleInstanceId": "final_norm", "moduleTemplateId": "model::RMSNorm", "isGroup": true, "moduleMembers": [ "final_norm.gain", "mul@final_norm#0" ], "moduleLayerCount": 2 } }, { "node_id": "lm_head", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 32, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "output", "kind": "output", "op": "Output", "params": {}, "in_ports": [ "x" ], "out_ports": [], "metadata": { "layerType": { "id": "output", "name": "Output", "displayName": "Output", "description": "Model output node", "icon": "ArrowLeft", "category": "io", "parameters": [ { "name": "name", "type": "string", "defaultValue": "output", "required": false, "description": "Name for this output" } ] } } }, { "node_id": "block.attention_norm", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block", "moduleInstanceId": "block.attention_norm", "moduleTemplateId": "model::RMSNorm", "isGroup": true, "moduleMembers": [ "block.attention_norm.gain", "mul@block.attention_norm#0" ], "moduleLayerCount": 2 } }, { "node_id": "block.attention", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block", "moduleInstanceId": "block.attention", "moduleTemplateId": "model::Attention", "isGroup": true, "moduleMembers": [ "python@block.attention#1", "block.attention.query", "block.attention.key", "block.attention.value", "python@block.attention#0", "block.attention.query_rope", "block.attention.key_rope", "div@block.attention#0", "block.attention.softmax", "block.attention.project" ], "moduleLayerCount": 32 } }, { "node_id": "add@block#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block" } }, { "node_id": "block.ffn_norm", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block", "moduleInstanceId": "block.ffn_norm", "moduleTemplateId": "model::RMSNorm", "isGroup": true, "moduleMembers": [ "block.ffn_norm.gain", "mul@block.ffn_norm#0" ], "moduleLayerCount": 2 } }, { "node_id": "block.ffn", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block", "moduleInstanceId": "block.ffn", "moduleTemplateId": "model::SwiGLU", "isGroup": true, "moduleMembers": [ "block.ffn.gate", "block.ffn.up", "block.ffn.silu", "mul@block.ffn#0", "block.ffn.down" ], "moduleLayerCount": 5 } }, { "node_id": "add@block#1", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block" } }, { "node_id": "block.attention_norm.gain", "kind": "layer", "op": "Parameter", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "parameter", "description": "", "category": "misc", "icon": "Box", "parameters": [], "name": "Parameter", "displayName": "Parameter" }, "parentId": "block.attention_norm" } }, { "node_id": "mul@block.attention_norm#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention_norm" } }, { "node_id": "python@block.attention#1", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention" } }, { "node_id": "block.attention.query", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 16, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.attention" } }, { "node_id": "block.attention.key", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 16, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.attention" } }, { "node_id": "block.attention.value", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 16, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.attention" } }, { "node_id": "python@block.attention#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention" } }, { "node_id": "block.attention.query_rope", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block.attention", "moduleInstanceId": "block.attention.query_rope", "moduleTemplateId": "model::Rotary", "isGroup": true, "moduleMembers": [ "python@block.attention.query_rope#2", "python@block.attention.query_rope#1", "python@block.attention.query_rope#0", "mul@block.attention.query_rope#0", "mul@block.attention.query_rope#1", "mul@block.attention.query_rope#2", "add@block.attention.query_rope#0", "mul@block.attention.query_rope#3", "mul@block.attention.query_rope#4", "sub@block.attention.query_rope#0", "stack@block.attention.query_rope#0", "flatten@block.attention.query_rope#0" ], "moduleLayerCount": 12 } }, { "node_id": "block.attention.key_rope", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "parentId": "block.attention", "moduleInstanceId": "block.attention.key_rope", "moduleTemplateId": "model::Rotary", "isGroup": true, "moduleMembers": [ "python@block.attention.key_rope#2", "python@block.attention.key_rope#1", "python@block.attention.key_rope#0", "mul@block.attention.key_rope#0", "mul@block.attention.key_rope#1", "mul@block.attention.key_rope#2", "add@block.attention.key_rope#0", "mul@block.attention.key_rope#3", "mul@block.attention.key_rope#4", "sub@block.attention.key_rope#0", "stack@block.attention.key_rope#0", "flatten@block.attention.key_rope#0" ], "moduleLayerCount": 12 } }, { "node_id": "div@block.attention#0", "kind": "layer", "op": "torch.div", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "div", "name": "torch.div", "displayName": "torch.div", "description": "PyTorch div operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention" } }, { "node_id": "block.attention.softmax", "kind": "layer", "op": "Softmax", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "softmax", "name": "Softmax", "displayName": "Softmax", "description": "Softmax over a dimension", "icon": "TrendingUp", "category": "activation", "parameters": [ { "name": "dim", "type": "number", "defaultValue": 1, "required": true, "description": "Dimension along which Softmax is computed" } ] }, "parentId": "block.attention" } }, { "node_id": "block.attention.project", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 16, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.attention" } }, { "node_id": "python@block.attention.query_rope#2", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.query_rope" } }, { "node_id": "python@block.attention.query_rope#1", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.query_rope" } }, { "node_id": "python@block.attention.query_rope#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.query_rope" } }, { "node_id": "mul@block.attention.query_rope#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "mul@block.attention.query_rope#1", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "mul@block.attention.query_rope#2", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "add@block.attention.query_rope#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "mul@block.attention.query_rope#3", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "mul@block.attention.query_rope#4", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "sub@block.attention.query_rope#0", "kind": "layer", "op": "torch.sub", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "sub", "name": "torch.sub", "displayName": "torch.sub", "description": "PyTorch sub operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "stack@block.attention.query_rope#0", "kind": "layer", "op": "torch.stack", "params": { "dim": -1 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "stack", "name": "torch.stack", "displayName": "torch.stack", "description": "PyTorch stack operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "flatten@block.attention.query_rope#0", "kind": "layer", "op": "Flatten", "params": { "start_dim": -2 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "flatten", "name": "Flatten", "displayName": "Flatten", "description": "Flattens input tensor contiguously", "icon": "Maximize2", "category": "tensor", "parameters": [ { "name": "start_dim", "type": "number", "defaultValue": 1, "required": false, "description": "First dim to flatten", "min": 0 }, { "name": "end_dim", "type": "number", "defaultValue": -1, "required": false, "description": "Last dim to flatten" } ] }, "parentId": "block.attention.query_rope" } }, { "node_id": "python@block.attention.key_rope#2", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.key_rope" } }, { "node_id": "python@block.attention.key_rope#1", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.key_rope" } }, { "node_id": "python@block.attention.key_rope#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "block.attention.key_rope" } }, { "node_id": "mul@block.attention.key_rope#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "mul@block.attention.key_rope#1", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "mul@block.attention.key_rope#2", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "add@block.attention.key_rope#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "mul@block.attention.key_rope#3", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "mul@block.attention.key_rope#4", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "sub@block.attention.key_rope#0", "kind": "layer", "op": "torch.sub", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "sub", "name": "torch.sub", "displayName": "torch.sub", "description": "PyTorch sub operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "stack@block.attention.key_rope#0", "kind": "layer", "op": "torch.stack", "params": { "dim": -1 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "stack", "name": "torch.stack", "displayName": "torch.stack", "description": "PyTorch stack operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "flatten@block.attention.key_rope#0", "kind": "layer", "op": "Flatten", "params": { "start_dim": -2 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "flatten", "name": "Flatten", "displayName": "Flatten", "description": "Flattens input tensor contiguously", "icon": "Maximize2", "category": "tensor", "parameters": [ { "name": "start_dim", "type": "number", "defaultValue": 1, "required": false, "description": "First dim to flatten", "min": 0 }, { "name": "end_dim", "type": "number", "defaultValue": -1, "required": false, "description": "Last dim to flatten" } ] }, "parentId": "block.attention.key_rope" } }, { "node_id": "block.ffn_norm.gain", "kind": "layer", "op": "Parameter", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "parameter", "description": "", "category": "misc", "icon": "Box", "parameters": [], "name": "Parameter", "displayName": "Parameter" }, "parentId": "block.ffn_norm" } }, { "node_id": "mul@block.ffn_norm#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.ffn_norm" } }, { "node_id": "block.ffn.gate", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 40, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.ffn" } }, { "node_id": "block.ffn.up", "kind": "layer", "op": "Linear", "params": { "in_features": 16, "out_features": 40, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.ffn" } }, { "node_id": "block.ffn.silu", "kind": "layer", "op": "SiLU", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "silu", "name": "SiLU", "displayName": "SiLU", "description": "Sigmoid Linear Unit (Swish)", "icon": "TrendingUp", "category": "activation", "parameters": [ { "name": "inplace", "type": "boolean", "defaultValue": false, "required": false, "description": "Can optionally do the operation in-place" } ] }, "parentId": "block.ffn" } }, { "node_id": "mul@block.ffn#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "block.ffn" } }, { "node_id": "block.ffn.down", "kind": "layer", "op": "Linear", "params": { "in_features": 40, "out_features": 16, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] }, "parentId": "block.ffn" } }, { "node_id": "final_norm.gain", "kind": "layer", "op": "Parameter", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "parameter", "description": "", "category": "misc", "icon": "Box", "parameters": [], "name": "Parameter", "displayName": "Parameter" }, "parentId": "final_norm" } }, { "node_id": "mul@final_norm#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "final_norm" } } ], "edges": [ { "edge_id": "edge-1", "source_node_id": "block.attention_norm.gain", "target_node_id": "mul@block.attention_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-2", "source_node_id": "python@block.attention.query_rope#0", "target_node_id": "mul@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-3", "source_node_id": "python@block.attention.query_rope#1", "target_node_id": "mul@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-4", "source_node_id": "python@block.attention.query_rope#2", "target_node_id": "mul@block.attention.query_rope#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-5", "source_node_id": "mul@block.attention.query_rope#0", "target_node_id": "mul@block.attention.query_rope#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-6", "source_node_id": "python@block.attention.query_rope#2", "target_node_id": "mul@block.attention.query_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-7", "source_node_id": "mul@block.attention.query_rope#0", "target_node_id": "mul@block.attention.query_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-8", "source_node_id": "mul@block.attention.query_rope#4", "target_node_id": "sub@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-9", "source_node_id": "mul@block.attention.query_rope#3", "target_node_id": "sub@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-10", "source_node_id": "python@block.attention.query_rope#2", "target_node_id": "mul@block.attention.query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-11", "source_node_id": "mul@block.attention.query_rope#0", "target_node_id": "mul@block.attention.query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-12", "source_node_id": "python@block.attention.query_rope#2", "target_node_id": "mul@block.attention.query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-13", "source_node_id": "mul@block.attention.query_rope#0", "target_node_id": "mul@block.attention.query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-14", "source_node_id": "mul@block.attention.query_rope#2", "target_node_id": "add@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-15", "source_node_id": "mul@block.attention.query_rope#1", "target_node_id": "add@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-16", "source_node_id": "sub@block.attention.query_rope#0", "target_node_id": "stack@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-17", "source_node_id": "add@block.attention.query_rope#0", "target_node_id": "stack@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-18", "source_node_id": "stack@block.attention.query_rope#0", "target_node_id": "flatten@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-19", "source_node_id": "python@block.attention.key_rope#0", "target_node_id": "mul@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-20", "source_node_id": "python@block.attention.key_rope#1", "target_node_id": "mul@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-21", "source_node_id": "python@block.attention.key_rope#2", "target_node_id": "mul@block.attention.key_rope#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-22", "source_node_id": "mul@block.attention.key_rope#0", "target_node_id": "mul@block.attention.key_rope#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-23", "source_node_id": "python@block.attention.key_rope#2", "target_node_id": "mul@block.attention.key_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-24", "source_node_id": "mul@block.attention.key_rope#0", "target_node_id": "mul@block.attention.key_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-25", "source_node_id": "mul@block.attention.key_rope#4", "target_node_id": "sub@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-26", "source_node_id": "mul@block.attention.key_rope#3", "target_node_id": "sub@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-27", "source_node_id": "python@block.attention.key_rope#2", "target_node_id": "mul@block.attention.key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-28", "source_node_id": "mul@block.attention.key_rope#0", "target_node_id": "mul@block.attention.key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-29", "source_node_id": "python@block.attention.key_rope#2", "target_node_id": "mul@block.attention.key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-30", "source_node_id": "mul@block.attention.key_rope#0", "target_node_id": "mul@block.attention.key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-31", "source_node_id": "mul@block.attention.key_rope#2", "target_node_id": "add@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-32", "source_node_id": "mul@block.attention.key_rope#1", "target_node_id": "add@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-33", "source_node_id": "sub@block.attention.key_rope#0", "target_node_id": "stack@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-34", "source_node_id": "add@block.attention.key_rope#0", "target_node_id": "stack@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-35", "source_node_id": "stack@block.attention.key_rope#0", "target_node_id": "flatten@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-36", "source_node_id": "block.attention.query", "target_node_id": "python@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-37", "source_node_id": "block.attention.query", "target_node_id": "python@block.attention.query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-38", "source_node_id": "block.attention.query", "target_node_id": "python@block.attention.query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-39", "source_node_id": "block.attention.query", "target_node_id": "block.attention.query_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-40", "source_node_id": "block.attention.key", "target_node_id": "python@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-41", "source_node_id": "block.attention.key", "target_node_id": "python@block.attention.key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-42", "source_node_id": "block.attention.key", "target_node_id": "python@block.attention.key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-43", "source_node_id": "block.attention.key", "target_node_id": "block.attention.key_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-44", "source_node_id": "block.attention.value", "target_node_id": "python@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-45", "source_node_id": "flatten@block.attention.query_rope#0", "target_node_id": "div@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-46", "source_node_id": "block.attention.query_rope", "target_node_id": "div@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-47", "source_node_id": "flatten@block.attention.key_rope#0", "target_node_id": "div@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-48", "source_node_id": "block.attention.key_rope", "target_node_id": "div@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-49", "source_node_id": "div@block.attention#0", "target_node_id": "block.attention.softmax", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-50", "source_node_id": "python@block.attention#1", "target_node_id": "block.attention.softmax", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-51", "source_node_id": "block.attention.softmax", "target_node_id": "block.attention.project", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-52", "source_node_id": "python@block.attention#0", "target_node_id": "block.attention.project", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-53", "source_node_id": "block.ffn_norm.gain", "target_node_id": "mul@block.ffn_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-54", "source_node_id": "block.ffn.gate", "target_node_id": "block.ffn.silu", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-55", "source_node_id": "block.ffn.silu", "target_node_id": "mul@block.ffn#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-56", "source_node_id": "block.ffn.up", "target_node_id": "mul@block.ffn#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-57", "source_node_id": "mul@block.ffn#0", "target_node_id": "block.ffn.down", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-58", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "block.attention.query", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-59", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-60", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-61", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-62", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "block.attention.key", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-63", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-64", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-65", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention.key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-66", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "block.attention.value", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-67", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-68", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "python@block.attention#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-69", "source_node_id": "mul@block.attention_norm#0", "target_node_id": "block.attention.project", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-70", "source_node_id": "block.attention_norm", "target_node_id": "block.attention", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-71", "source_node_id": "block.attention.project", "target_node_id": "add@block#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-72", "source_node_id": "block.attention", "target_node_id": "add@block#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-73", "source_node_id": "add@block#0", "target_node_id": "mul@block.ffn_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-74", "source_node_id": "add@block#0", "target_node_id": "block.ffn_norm", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-75", "source_node_id": "mul@block.ffn_norm#0", "target_node_id": "block.ffn.gate", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-76", "source_node_id": "mul@block.ffn_norm#0", "target_node_id": "block.ffn.up", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-77", "source_node_id": "block.ffn_norm", "target_node_id": "block.ffn", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-78", "source_node_id": "add@block#0", "target_node_id": "add@block#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-79", "source_node_id": "block.ffn.down", "target_node_id": "add@block#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-80", "source_node_id": "block.ffn", "target_node_id": "add@block#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-81", "source_node_id": "final_norm.gain", "target_node_id": "mul@final_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-82", "source_node_id": "input", "target_node_id": "embedding", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-83", "source_node_id": "embedding", "target_node_id": "mul@block.attention_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-84", "source_node_id": "embedding", "target_node_id": "add@block#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-85", "source_node_id": "embedding", "target_node_id": "block", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-86", "source_node_id": "add@block#1", "target_node_id": "mul@final_norm#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-87", "source_node_id": "block", "target_node_id": "final_norm", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-88", "source_node_id": "mul@final_norm#0", "target_node_id": "lm_head", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-89", "source_node_id": "final_norm", "target_node_id": "lm_head", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-90", "source_node_id": "lm_head", "target_node_id": "output", "source_port": "y", "target_port": "x", "metadata": {} } ], "inputs": [], "modules": [], "operations": [], "metadata": {} }, "view": { "graph_id": "snapshot", "node_positions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "collapsed_modules": [ "block", "final_norm", "block.attention_norm", "block.attention", "block.ffn_norm", "block.ffn", "block.attention.query_rope", "block.attention.key_rope" ], "selection": [], "layout": {} } }, "snapshotId": "snapshot-6a65dc3ff0899b95", "collaboration": { "walkthrough": { "title": "Bring the modern decoder pieces together", "steps": [ { "id": "tokens", "title": "Map integer tokens into model features", "note": "The input contains six integer IDs from a vocabulary of 32. Each becomes a 16-feature vector. There are no learned absolute position embeddings; position enters through attention's RoPE.", "view": { "camera": { "x": -7.5, "y": 60, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "expandedModuleIds": [], "selectedNodeIds": [ "embedding" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "embedding" }, "snapshotId": "snapshot-6a65dc3ff0899b95", "exportedAt": "2026-09-17T06:13:04.092484+00:00", "modelName": "TinyLlama", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"embedding\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@block#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"embedding\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"embedding\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"embedding\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"embedding\",\"kind\":\"layer\",\"op\":\"Embedding\",\"params\":{\"embedding_dim\":16,\"num_embeddings\":32}}}" } ] }, { "id": "pre-norm", "title": "Normalize inside the residual branch", "note": "RMSNorm runs before attention. The original features bypass it on the residual path, so zeroing the branch's output projection leaves the block input available unchanged.", "view": { "camera": { "x": 291, "y": -4, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "expandedModuleIds": [ "block" ], "selectedNodeIds": [ "block.attention_norm" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "block.attention_norm" }, "snapshotId": "snapshot-6a65dc3ff0899b95", "exportedAt": "2026-09-17T06:13:04.092484+00:00", "modelName": "TinyLlama", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention_norm\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention_norm.gain\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"embedding\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.key\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.project\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.query\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.value\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"block.attention_norm.gain\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Parameter\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention_norm\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention_norm#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention_norm\\\"}\"],\"node\":{\"id\":\"block.attention_norm\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{},\"parent\":\"block\"}}" } ] }, { "id": "rotary", "title": "Add position to causal attention", "note": "Four independent heads rotate their query/key pairs. The causal mask hides later tokens; values are mixed without rotation. This combines two different mechanisms: order information and future-token visibility.", "view": { "camera": { "x": 1732.923076923077, "y": -739.7762237762238, "z": 0.28542914171656686 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "expandedModuleIds": [ "block", "block.attention", "block.attention.query_rope" ], "selectedNodeIds": [ "block.attention.query_rope" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "block.attention.query_rope" }, "snapshotId": "snapshot-6a65dc3ff0899b95", "exportedAt": "2026-09-17T06:13:04.092484+00:00", "modelName": "TinyLlama", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"add@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention.query\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.query_rope\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention.query\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention.query\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention.query\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention.query_rope\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"div@block.attention#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"flatten@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"div@block.attention#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#3\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#4\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#1\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#3\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention.query_rope#4\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#1\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#3\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@block.attention.query_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention.query_rope#4\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"stack@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"flatten@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"sub@block.attention.query_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"add@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.add\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"flatten@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Flatten\\\",\\\"params\\\":{\\\"start_dim\\\":-2},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention.query_rope#1\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention.query_rope#2\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention.query_rope#3\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention.query_rope#4\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"python@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Python\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"python@block.attention.query_rope#1\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Python\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"python@block.attention.query_rope#2\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Python\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"stack@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.stack\\\",\\\"params\\\":{\\\"dim\\\":-1},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\",\"{\\\"id\\\":\\\"sub@block.attention.query_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.sub\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention.query_rope\\\"}\"],\"node\":{\"id\":\"block.attention.query_rope\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{},\"parent\":\"block.attention\"}}" } ] }, { "id": "gated", "title": "Apply a second pre-norm and a SwiGLU branch", "note": "After attention rejoins the residual, another RMSNorm feeds a gated FFN. Its output is added to the running residual stream. A final RMSNorm and vocabulary head then produce 32 logits per token.", "view": { "camera": { "x": 730.0384615384615, "y": -706.4755244755245, "z": 0.595213319458897 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "expandedModuleIds": [ "block", "block.ffn" ], "selectedNodeIds": [ "block.ffn" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "block.ffn" }, "snapshotId": "snapshot-6a65dc3ff0899b95", "exportedAt": "2026-09-17T06:13:04.092484+00:00", "modelName": "TinyLlama", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@block#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn.down\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@block#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn.gate\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.ffn.silu\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn.silu\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.ffn#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn.up\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.ffn#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.ffn_norm\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.ffn\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.ffn#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.ffn.down\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.ffn_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.ffn.gate\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.ffn_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.ffn.up\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"block.ffn.down\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Linear\\\",\\\"params\\\":{\\\"bias\\\":false,\\\"in_features\\\":40,\\\"out_features\\\":16},\\\"parent\\\":\\\"block.ffn\\\"}\",\"{\\\"id\\\":\\\"block.ffn.gate\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Linear\\\",\\\"params\\\":{\\\"bias\\\":false,\\\"in_features\\\":16,\\\"out_features\\\":40},\\\"parent\\\":\\\"block.ffn\\\"}\",\"{\\\"id\\\":\\\"block.ffn.silu\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"SiLU\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.ffn\\\"}\",\"{\\\"id\\\":\\\"block.ffn.up\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Linear\\\",\\\"params\\\":{\\\"bias\\\":false,\\\"in_features\\\":16,\\\"out_features\\\":40},\\\"parent\\\":\\\"block.ffn\\\"}\",\"{\\\"id\\\":\\\"mul@block.ffn#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.ffn\\\"}\"],\"node\":{\"id\":\"block.ffn\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{},\"parent\":\"block\"}}" } ] } ] }, "threads": [], "savedViews": [ { "id": "pre-norm", "title": "Normalize inside the residual branch", "note": "RMSNorm runs before attention. The original features bypass it on the residual path, so zeroing the branch's output projection leaves the block input available unchanged.", "view": { "camera": { "x": 291, "y": -4, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "embedding": { "x": 340, "y": 160 }, "block": { "x": 680, "y": 160 }, "final_norm": { "x": 1020, "y": 160 }, "lm_head": { "x": 1360, "y": 160 }, "output": { "x": 1700, "y": 160 }, "block.attention_norm": { "x": 24, "y": 214 }, "block.attention": { "x": 24, "y": 364 }, "add@block#0": { "x": 24, "y": 514 }, "block.ffn_norm": { "x": 24, "y": 664 }, "block.ffn": { "x": 24, "y": 814 }, "add@block#1": { "x": 24, "y": 964 }, "block.attention_norm.gain": { "x": 24, "y": 214 }, "mul@block.attention_norm#0": { "x": 24, "y": 364 }, "python@block.attention#1": { "x": 24, "y": 214 }, "block.attention.query": { "x": 24, "y": 364 }, "block.attention.key": { "x": 24, "y": 514 }, "block.attention.value": { "x": 24, "y": 664 }, "python@block.attention#0": { "x": 24, "y": 814 }, "block.attention.query_rope": { "x": 24, "y": 964 }, "block.attention.key_rope": { "x": 24, "y": 1114 }, "div@block.attention#0": { "x": 24, "y": 1264 }, "block.attention.softmax": { "x": 24, "y": 1414 }, "block.attention.project": { "x": 24, "y": 1564 }, "python@block.attention.query_rope#2": { "x": 24, "y": 214 }, "python@block.attention.query_rope#1": { "x": 24, "y": 364 }, "python@block.attention.query_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.query_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.query_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.query_rope#2": { "x": 24, "y": 964 }, "add@block.attention.query_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.query_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.query_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.query_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.query_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.query_rope#0": { "x": 24, "y": 1864 }, "python@block.attention.key_rope#2": { "x": 24, "y": 214 }, "python@block.attention.key_rope#1": { "x": 24, "y": 364 }, "python@block.attention.key_rope#0": { "x": 24, "y": 514 }, "mul@block.attention.key_rope#0": { "x": 24, "y": 664 }, "mul@block.attention.key_rope#1": { "x": 24, "y": 814 }, "mul@block.attention.key_rope#2": { "x": 24, "y": 964 }, "add@block.attention.key_rope#0": { "x": 24, "y": 1114 }, "mul@block.attention.key_rope#3": { "x": 24, "y": 1264 }, "mul@block.attention.key_rope#4": { "x": 24, "y": 1414 }, "sub@block.attention.key_rope#0": { "x": 24, "y": 1564 }, "stack@block.attention.key_rope#0": { "x": 24, "y": 1714 }, "flatten@block.attention.key_rope#0": { "x": 24, "y": 1864 }, "block.ffn_norm.gain": { "x": 24, "y": 214 }, "mul@block.ffn_norm#0": { "x": 24, "y": 364 }, "block.ffn.gate": { "x": 24, "y": 214 }, "block.ffn.up": { "x": 24, "y": 364 }, "block.ffn.silu": { "x": 24, "y": 514 }, "mul@block.ffn#0": { "x": 24, "y": 664 }, "block.ffn.down": { "x": 24, "y": 814 }, "final_norm.gain": { "x": 24, "y": 214 }, "mul@final_norm#0": { "x": 24, "y": 364 } }, "expandedModuleIds": [ "block" ], "selectedNodeIds": [ "block.attention_norm" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "block.attention_norm" }, "snapshotId": "snapshot-6a65dc3ff0899b95", "exportedAt": "2026-09-17T06:13:04.092484+00:00", "modelName": "TinyLlama", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention_norm\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"block.attention_norm.gain\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"embedding\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@block.attention_norm#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.key\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.project\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.query\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"block.attention.value\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.key_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@block.attention_norm#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@block.attention.query_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"block.attention_norm.gain\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Parameter\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention_norm\\\"}\",\"{\\\"id\\\":\\\"mul@block.attention_norm#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"block.attention_norm\\\"}\"],\"node\":{\"id\":\"block.attention_norm\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{},\"parent\":\"block\"}}" } ] } ], "openingViewId": "pre-norm" } }