{ "format": "tensorviz.graph", "version": 3, "exportedAt": "2026-09-17T06:13:00.308801+00:00", "status": "static", "warningCount": 0, "document": { "schemaVersion": 2, "semantic": { "graph_id": "snapshot", "name": "LatentAttention", "revision": 0, "nodes": [ { "node_id": "input", "kind": "input", "op": "Input", "params": { "shape": "1,6,8" }, "in_ports": [], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "input", "name": "Input", "displayName": "Input", "description": "Model input node", "icon": "ArrowRight", "category": "io", "parameters": [ { "name": "shape", "type": "string", "defaultValue": "1,3,224,224", "required": true, "description": "Input tensor shape (comma-separated, e.g., 1,3,224,224)" }, { "name": "name", "type": "string", "defaultValue": "input", "required": false, "description": "Name for this input" } ] } } }, { "node_id": "python@LatentAttention#5", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "kv_latent", "kind": "layer", "op": "Linear", "params": { "in_features": 8, "out_features": 3, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "query_latent", "kind": "layer", "op": "Linear", "params": { "in_features": 8, "out_features": 4, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "query_content", "kind": "layer", "op": "Linear", "params": { "in_features": 4, "out_features": 8, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "python@LatentAttention#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "key_content", "kind": "layer", "op": "Linear", "params": { "in_features": 3, "out_features": 8, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "python@LatentAttention#1", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "value_content", "kind": "layer", "op": "Linear", "params": { "in_features": 3, "out_features": 8, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "python@LatentAttention#2", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "query_position", "kind": "layer", "op": "Linear", "params": { "in_features": 4, "out_features": 8, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "key_position", "kind": "layer", "op": "Linear", "params": { "in_features": 8, "out_features": 2, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "query_rope", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "moduleInstanceId": "query_rope", "moduleTemplateId": "model::PositionPair", "isGroup": true, "moduleMembers": [ "python@query_rope#0", "mul@query_rope#0", "mul@query_rope#1", "add@query_rope#0", "mul@query_rope#2", "mul@query_rope#3", "sub@query_rope#0", "stack@query_rope#0" ], "moduleLayerCount": 8 } }, { "node_id": "python@LatentAttention#3", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "key_rope", "kind": "module", "op": "Module", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "module", "name": "Module", "displayName": "Module", "description": "Custom module group (nn.Module)", "icon": "Package", "category": "module", "parameters": [ { "name": "name", "type": "string", "defaultValue": "CustomModule", "required": true, "description": "Module class name" }, { "name": "description", "type": "string", "defaultValue": "", "required": false, "description": "Module description" } ] }, "moduleInstanceId": "key_rope", "moduleTemplateId": "model::PositionPair", "isGroup": true, "moduleMembers": [ "python@key_rope#0", "mul@key_rope#0", "mul@key_rope#1", "add@key_rope#0", "mul@key_rope#2", "mul@key_rope#3", "sub@key_rope#0", "stack@key_rope#0" ], "moduleLayerCount": 8 } }, { "node_id": "python@LatentAttention#4", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] } } }, { "node_id": "add@LatentAttention#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] } } }, { "node_id": "softmax", "kind": "layer", "op": "Softmax", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "softmax", "name": "Softmax", "displayName": "Softmax", "description": "Softmax over a dimension", "icon": "TrendingUp", "category": "activation", "parameters": [ { "name": "dim", "type": "number", "defaultValue": 1, "required": true, "description": "Dimension along which Softmax is computed" } ] } } }, { "node_id": "output_projection", "kind": "layer", "op": "Linear", "params": { "in_features": 8, "out_features": 8, "bias": false }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "linear", "name": "Linear", "displayName": "Linear", "description": "Fully Connected Layer", "icon": "Minus", "category": "linear", "parameters": [ { "name": "in_features", "type": "number", "defaultValue": 512, "required": true, "description": "Size of input features", "min": 1 }, { "name": "out_features", "type": "number", "defaultValue": 256, "required": true, "description": "Size of output features", "min": 1 }, { "name": "bias", "type": "boolean", "defaultValue": true, "required": false, "description": "If set to False, the layer will not learn an additive bias" } ] } } }, { "node_id": "output", "kind": "output", "op": "Output", "params": {}, "in_ports": [ "x" ], "out_ports": [], "metadata": { "layerType": { "id": "output", "name": "Output", "displayName": "Output", "description": "Model output node", "icon": "ArrowLeft", "category": "io", "parameters": [ { "name": "name", "type": "string", "defaultValue": "output", "required": false, "description": "Name for this output" } ] } } }, { "node_id": "python@query_rope#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "query_rope" } }, { "node_id": "mul@query_rope#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "mul@query_rope#1", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "add@query_rope#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "mul@query_rope#2", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "mul@query_rope#3", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "sub@query_rope#0", "kind": "layer", "op": "torch.sub", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "sub", "name": "torch.sub", "displayName": "torch.sub", "description": "PyTorch sub operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "stack@query_rope#0", "kind": "layer", "op": "torch.stack", "params": { "dim": -1 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "stack", "name": "torch.stack", "displayName": "torch.stack", "description": "PyTorch stack operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "query_rope" } }, { "node_id": "python@key_rope#0", "kind": "layer", "op": "Python", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "python", "name": "Python", "displayName": "Python", "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.", "icon": "Hash", "category": "misc", "parameters": [] }, "parentId": "key_rope" } }, { "node_id": "mul@key_rope#0", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "mul@key_rope#1", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "add@key_rope#0", "kind": "layer", "op": "torch.add", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "add", "name": "torch.add", "displayName": "torch.add", "description": "PyTorch add operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "mul@key_rope#2", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "mul@key_rope#3", "kind": "layer", "op": "torch.mul", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "mul", "name": "torch.mul", "displayName": "torch.mul", "description": "PyTorch mul operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "sub@key_rope#0", "kind": "layer", "op": "torch.sub", "params": {}, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "sub", "name": "torch.sub", "displayName": "torch.sub", "description": "PyTorch sub operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } }, { "node_id": "stack@key_rope#0", "kind": "layer", "op": "torch.stack", "params": { "dim": -1 }, "in_ports": [ "x" ], "out_ports": [ "y" ], "metadata": { "layerType": { "id": "stack", "name": "torch.stack", "displayName": "torch.stack", "description": "PyTorch stack operation", "icon": "Activity", "category": "tensor", "parameters": [ { "name": "dim", "type": "string", "defaultValue": "", "required": false, "description": "Dimension(s) to apply the operation on" } ] }, "parentId": "key_rope" } } ], "edges": [ { "edge_id": "edge-1", "source_node_id": "python@query_rope#0", "target_node_id": "mul@query_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-2", "source_node_id": "python@query_rope#0", "target_node_id": "mul@query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-3", "source_node_id": "mul@query_rope#3", "target_node_id": "sub@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-4", "source_node_id": "mul@query_rope#2", "target_node_id": "sub@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-5", "source_node_id": "python@query_rope#0", "target_node_id": "mul@query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-6", "source_node_id": "python@query_rope#0", "target_node_id": "mul@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-7", "source_node_id": "mul@query_rope#1", "target_node_id": "add@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-8", "source_node_id": "mul@query_rope#0", "target_node_id": "add@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-9", "source_node_id": "sub@query_rope#0", "target_node_id": "stack@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-10", "source_node_id": "add@query_rope#0", "target_node_id": "stack@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-11", "source_node_id": "python@key_rope#0", "target_node_id": "mul@key_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-12", "source_node_id": "python@key_rope#0", "target_node_id": "mul@key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-13", "source_node_id": "mul@key_rope#3", "target_node_id": "sub@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-14", "source_node_id": "mul@key_rope#2", "target_node_id": "sub@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-15", "source_node_id": "python@key_rope#0", "target_node_id": "mul@key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-16", "source_node_id": "python@key_rope#0", "target_node_id": "mul@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-17", "source_node_id": "mul@key_rope#1", "target_node_id": "add@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-18", "source_node_id": "mul@key_rope#0", "target_node_id": "add@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-19", "source_node_id": "sub@key_rope#0", "target_node_id": "stack@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-20", "source_node_id": "add@key_rope#0", "target_node_id": "stack@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-21", "source_node_id": "input", "target_node_id": "kv_latent", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-22", "source_node_id": "input", "target_node_id": "query_latent", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-23", "source_node_id": "query_latent", "target_node_id": "query_content", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-24", "source_node_id": "query_content", "target_node_id": "python@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-25", "source_node_id": "input", "target_node_id": "python@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-26", "source_node_id": "kv_latent", "target_node_id": "key_content", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-27", "source_node_id": "key_content", "target_node_id": "python@LatentAttention#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-28", "source_node_id": "input", "target_node_id": "python@LatentAttention#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-29", "source_node_id": "kv_latent", "target_node_id": "value_content", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-30", "source_node_id": "value_content", "target_node_id": "python@LatentAttention#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-31", "source_node_id": "input", "target_node_id": "python@LatentAttention#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-32", "source_node_id": "query_latent", "target_node_id": "query_position", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-33", "source_node_id": "query_position", "target_node_id": "python@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-34", "source_node_id": "query_position", "target_node_id": "mul@query_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-35", "source_node_id": "query_position", "target_node_id": "mul@query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-36", "source_node_id": "query_position", "target_node_id": "mul@query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-37", "source_node_id": "query_position", "target_node_id": "mul@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-38", "source_node_id": "query_position", "target_node_id": "query_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-39", "source_node_id": "input", "target_node_id": "python@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-40", "source_node_id": "input", "target_node_id": "mul@query_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-41", "source_node_id": "input", "target_node_id": "mul@query_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-42", "source_node_id": "input", "target_node_id": "mul@query_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-43", "source_node_id": "input", "target_node_id": "mul@query_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-44", "source_node_id": "input", "target_node_id": "query_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-45", "source_node_id": "stack@query_rope#0", "target_node_id": "python@LatentAttention#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-46", "source_node_id": "query_rope", "target_node_id": "python@LatentAttention#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-47", "source_node_id": "input", "target_node_id": "key_position", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-48", "source_node_id": "key_position", "target_node_id": "python@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-49", "source_node_id": "key_position", "target_node_id": "mul@key_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-50", "source_node_id": "key_position", "target_node_id": "mul@key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-51", "source_node_id": "key_position", "target_node_id": "mul@key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-52", "source_node_id": "key_position", "target_node_id": "mul@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-53", "source_node_id": "key_position", "target_node_id": "key_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-54", "source_node_id": "input", "target_node_id": "python@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-55", "source_node_id": "input", "target_node_id": "mul@key_rope#3", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-56", "source_node_id": "input", "target_node_id": "mul@key_rope#2", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-57", "source_node_id": "input", "target_node_id": "mul@key_rope#1", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-58", "source_node_id": "input", "target_node_id": "mul@key_rope#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-59", "source_node_id": "input", "target_node_id": "key_rope", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-60", "source_node_id": "stack@key_rope#0", "target_node_id": "python@LatentAttention#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-61", "source_node_id": "key_rope", "target_node_id": "python@LatentAttention#4", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-62", "source_node_id": "python@LatentAttention#0", "target_node_id": "add@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-63", "source_node_id": "python@LatentAttention#1", "target_node_id": "add@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-64", "source_node_id": "python@LatentAttention#3", "target_node_id": "add@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-65", "source_node_id": "python@LatentAttention#4", "target_node_id": "add@LatentAttention#0", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-66", "source_node_id": "input", "target_node_id": "python@LatentAttention#5", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-67", "source_node_id": "add@LatentAttention#0", "target_node_id": "softmax", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-68", "source_node_id": "python@LatentAttention#5", "target_node_id": "softmax", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-69", "source_node_id": "softmax", "target_node_id": "output_projection", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-70", "source_node_id": "python@LatentAttention#2", "target_node_id": "output_projection", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-71", "source_node_id": "input", "target_node_id": "output_projection", "source_port": "y", "target_port": "x", "metadata": {} }, { "edge_id": "edge-72", "source_node_id": "output_projection", "target_node_id": "output", "source_port": "y", "target_port": "x", "metadata": {} } ], "inputs": [], "modules": [], "operations": [], "metadata": {} }, "view": { "graph_id": "snapshot", "node_positions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "collapsed_modules": [ "query_rope", "key_rope" ], "selection": [], "layout": {} } }, "snapshotId": "snapshot-9233b8c9e79d837f", "collaboration": { "walkthrough": { "title": "Compress the cache, keep position separate", "steps": [ { "id": "compress", "title": "Create one shared latent for K and V", "note": "An 8-to-3 projection produces the joint KV latent. Content-key and value up-projections reconstruct four heads from it in this explicit forward.", "view": { "camera": { "x": -347.5, "y": 56.5, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "expandedModuleIds": [], "selectedNodeIds": [ "kv_latent" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "kv_latent" }, "snapshotId": "snapshot-9233b8c9e79d837f", "exportedAt": "2026-09-17T06:13:00.308801+00:00", "modelName": "LatentAttention", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"kv_latent\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"kv_latent\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"key_content\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"kv_latent\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"value_content\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"kv_latent\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":8,\"out_features\":3}}}" } ] }, { "id": "query", "title": "Compress and expand the current query", "note": "The query has its own 8-to-4 down-projection and a four-to-eight content up-projection. Query latents are used for current queries and do not become part of the prefix KV cache.", "view": { "camera": { "x": -1027.5, "y": 56.5, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "expandedModuleIds": [], "selectedNodeIds": [ "query_content" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "query_content" }, "snapshotId": "snapshot-9233b8c9e79d837f", "exportedAt": "2026-09-17T06:13:00.308801+00:00", "modelName": "LatentAttention", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"query_content\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@LatentAttention#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"query_latent\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"query_content\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"query_content\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":4,\"out_features\":8}}}" } ] }, { "id": "position", "title": "Cache a separate positional key", "note": "RoPE acts on an additional shared width-two key and per-head positional queries. Keeping these features separate lets content-key weights be absorbed into the query calculation without moving them across token-dependent rotations.", "view": { "camera": { "x": -3580, "y": -2.909090909090878, "z": 0.4074074074074074 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "expandedModuleIds": [ "key_rope" ], "selectedNodeIds": [ "key_rope" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "key_rope" }, "snapshotId": "snapshot-9233b8c9e79d837f", "exportedAt": "2026-09-17T06:13:00.308801+00:00", "modelName": "LatentAttention", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"add@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"key_rope\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#3\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"key_rope\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#3\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_position\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"key_rope\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@LatentAttention#4\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@key_rope#1\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"add@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@key_rope#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"mul@key_rope#3\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#1\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#2\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"mul@key_rope#3\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"stack@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@LatentAttention#4\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"sub@key_rope#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"stack@key_rope#0\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"add@key_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.add\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"mul@key_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"mul@key_rope#1\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"mul@key_rope#2\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"mul@key_rope#3\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.mul\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"python@key_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Python\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"stack@key_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.stack\\\",\\\"params\\\":{\\\"dim\\\":-1},\\\"parent\\\":\\\"key_rope\\\"}\",\"{\\\"id\\\":\\\"sub@key_rope#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"torch.sub\\\",\\\"params\\\":{},\\\"parent\\\":\\\"key_rope\\\"}\"],\"node\":{\"id\":\"key_rope\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}" } ] }, { "id": "absorb", "title": "Absorb value reconstruction into the output", "note": "During compact inference, attention mixes latent vectors. Combining value-up and output-projection weights then gives the same result without expanding cached values. Compare this independent calculation below.", "view": { "camera": { "x": -5787.5, "y": 56.5, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "expandedModuleIds": [], "selectedNodeIds": [ "output_projection" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "output_projection" }, "snapshotId": "snapshot-9233b8c9e79d837f", "exportedAt": "2026-09-17T06:13:00.308801+00:00", "modelName": "LatentAttention", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"output_projection\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"output_projection\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"output\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@LatentAttention#2\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"output_projection\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"softmax\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"output_projection\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"output_projection\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":8,\"out_features\":8}}}" } ] } ] }, "threads": [], "savedViews": [ { "id": "compress", "title": "Create one shared latent for K and V", "note": "An 8-to-3 projection produces the joint KV latent. Content-key and value up-projections reconstruct four heads from it in this explicit forward.", "view": { "camera": { "x": -347.5, "y": 56.5, "z": 1.25 }, "nodePositions": { "input": { "x": 0, "y": 160 }, "python@LatentAttention#5": { "x": 340, "y": 160 }, "kv_latent": { "x": 680, "y": 160 }, "query_latent": { "x": 1020, "y": 160 }, "query_content": { "x": 1360, "y": 160 }, "python@LatentAttention#0": { "x": 1700, "y": 160 }, "key_content": { "x": 2040, "y": 160 }, "python@LatentAttention#1": { "x": 2380, "y": 160 }, "value_content": { "x": 2720, "y": 160 }, "python@LatentAttention#2": { "x": 3060, "y": 160 }, "query_position": { "x": 3400, "y": 160 }, "key_position": { "x": 3740, "y": 160 }, "query_rope": { "x": 4080, "y": 160 }, "python@LatentAttention#3": { "x": 4420, "y": 160 }, "key_rope": { "x": 4760, "y": 160 }, "python@LatentAttention#4": { "x": 5100, "y": 160 }, "add@LatentAttention#0": { "x": 5440, "y": 160 }, "softmax": { "x": 5780, "y": 160 }, "output_projection": { "x": 6120, "y": 160 }, "output": { "x": 6460, "y": 160 }, "python@query_rope#0": { "x": 24, "y": 214 }, "mul@query_rope#0": { "x": 24, "y": 364 }, "mul@query_rope#1": { "x": 24, "y": 514 }, "add@query_rope#0": { "x": 24, "y": 664 }, "mul@query_rope#2": { "x": 24, "y": 814 }, "mul@query_rope#3": { "x": 24, "y": 964 }, "sub@query_rope#0": { "x": 24, "y": 1114 }, "stack@query_rope#0": { "x": 24, "y": 1264 }, "python@key_rope#0": { "x": 24, "y": 214 }, "mul@key_rope#0": { "x": 24, "y": 364 }, "mul@key_rope#1": { "x": 24, "y": 514 }, "add@key_rope#0": { "x": 24, "y": 664 }, "mul@key_rope#2": { "x": 24, "y": 814 }, "mul@key_rope#3": { "x": 24, "y": 964 }, "sub@key_rope#0": { "x": 24, "y": 1114 }, "stack@key_rope#0": { "x": 24, "y": 1264 } }, "expandedModuleIds": [], "selectedNodeIds": [ "kv_latent" ], "viewport": { "width": 1100, "height": 700 } }, "anchors": [ { "target": { "kind": "node", "nodeId": "kv_latent" }, "snapshotId": "snapshot-9233b8c9e79d837f", "exportedAt": "2026-09-17T06:13:00.308801+00:00", "modelName": "LatentAttention", "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"kv_latent\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"kv_latent\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"key_content\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"kv_latent\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"value_content\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"kv_latent\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"bias\":false,\"in_features\":8,\"out_features\":3}}}" } ] } ], "openingViewId": "compress" } }