{
  "format": "tensorviz.graph",
  "version": 3,
  "exportedAt": "2026-09-17T07:02:39.684264+00:00",
  "status": "static",
  "warningCount": 0,
  "document": {
    "schemaVersion": 2,
    "semantic": {
      "graph_id": "snapshot",
      "name": "ToyGRPO",
      "revision": 0,
      "nodes": [
        {
          "node_id": "input",
          "kind": "input",
          "op": "Input",
          "params": {
            "shape": "1,3,224,224"
          },
          "in_ports": [],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "input",
              "name": "Input",
              "displayName": "Input",
              "description": "Model input node",
              "icon": "ArrowRight",
              "category": "io",
              "parameters": [
                {
                  "name": "shape",
                  "type": "string",
                  "defaultValue": "1,3,224,224",
                  "required": true,
                  "description": "Input tensor shape (comma-separated, e.g., 1,3,224,224)"
                },
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "input",
                  "required": false,
                  "description": "Name for this input"
                }
              ]
            }
          }
        },
        {
          "node_id": "policy",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "policy",
            "moduleTemplateId": "model::CompletionPolicy",
            "isGroup": true,
            "moduleMembers": [
              "policy.logits",
              "policy.log_probabilities"
            ],
            "moduleLayerCount": 2
          }
        },
        {
          "node_id": "old_policy",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "old_policy",
            "moduleTemplateId": "model::CompletionPolicy",
            "isGroup": true,
            "moduleMembers": [
              "old_policy.logits",
              "old_policy.log_probabilities"
            ],
            "moduleLayerCount": 2
          }
        },
        {
          "node_id": "reference_policy",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "reference_policy",
            "moduleTemplateId": "model::CompletionPolicy",
            "isGroup": true,
            "moduleMembers": [
              "reference_policy.logits",
              "reference_policy.log_probabilities"
            ],
            "moduleLayerCount": 2
          }
        },
        {
          "node_id": "advantages",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "advantages",
            "moduleTemplateId": "model::GroupAdvantages",
            "isGroup": true,
            "moduleMembers": [],
            "moduleLayerCount": 0
          }
        },
        {
          "node_id": "ratio",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "ratio",
            "moduleTemplateId": "model::PolicyRatio",
            "isGroup": true,
            "moduleMembers": [
              "python@ratio#0"
            ],
            "moduleLayerCount": 1
          }
        },
        {
          "node_id": "clipped",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "clipped",
            "moduleTemplateId": "model::ClippedSurrogate",
            "isGroup": true,
            "moduleMembers": [
              "python@clipped#0"
            ],
            "moduleLayerCount": 1
          }
        },
        {
          "node_id": "divergence",
          "kind": "module",
          "op": "Module",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "module",
              "name": "Module",
              "displayName": "Module",
              "description": "Custom module group (nn.Module)",
              "icon": "Package",
              "category": "module",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "CustomModule",
                  "required": true,
                  "description": "Module class name"
                },
                {
                  "name": "description",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Module description"
                }
              ]
            },
            "moduleInstanceId": "divergence",
            "moduleTemplateId": "model::ReferencePenalty",
            "isGroup": true,
            "moduleMembers": [],
            "moduleLayerCount": 0
          }
        },
        {
          "node_id": "sub@ToyGRPO#0",
          "kind": "layer",
          "op": "torch.sub",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "sub",
              "name": "torch.sub",
              "displayName": "torch.sub",
              "description": "PyTorch sub operation",
              "icon": "Activity",
              "category": "tensor",
              "parameters": [
                {
                  "name": "dim",
                  "type": "string",
                  "defaultValue": "",
                  "required": false,
                  "description": "Dimension(s) to apply the operation on"
                }
              ]
            }
          }
        },
        {
          "node_id": "output",
          "kind": "output",
          "op": "Output",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [],
          "metadata": {
            "layerType": {
              "id": "output",
              "name": "Output",
              "displayName": "Output",
              "description": "Model output node",
              "icon": "ArrowLeft",
              "category": "io",
              "parameters": [
                {
                  "name": "name",
                  "type": "string",
                  "defaultValue": "output",
                  "required": false,
                  "description": "Name for this output"
                }
              ]
            }
          }
        },
        {
          "node_id": "policy.logits",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 3,
            "out_features": 4
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "policy"
          }
        },
        {
          "node_id": "policy.log_probabilities",
          "kind": "layer",
          "op": "LogSoftmax",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "logsoftmax",
              "name": "LogSoftmax",
              "displayName": "LogSoftmax",
              "description": "Log-Softmax over a dimension",
              "icon": "TrendingUp",
              "category": "activation",
              "parameters": [
                {
                  "name": "dim",
                  "type": "number",
                  "defaultValue": 1,
                  "required": true,
                  "description": "Dimension along which LogSoftmax is computed"
                }
              ]
            },
            "parentId": "policy"
          }
        },
        {
          "node_id": "old_policy.logits",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 3,
            "out_features": 4
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "old_policy"
          }
        },
        {
          "node_id": "old_policy.log_probabilities",
          "kind": "layer",
          "op": "LogSoftmax",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "logsoftmax",
              "name": "LogSoftmax",
              "displayName": "LogSoftmax",
              "description": "Log-Softmax over a dimension",
              "icon": "TrendingUp",
              "category": "activation",
              "parameters": [
                {
                  "name": "dim",
                  "type": "number",
                  "defaultValue": 1,
                  "required": true,
                  "description": "Dimension along which LogSoftmax is computed"
                }
              ]
            },
            "parentId": "old_policy"
          }
        },
        {
          "node_id": "reference_policy.logits",
          "kind": "layer",
          "op": "Linear",
          "params": {
            "in_features": 3,
            "out_features": 4
          },
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "linear",
              "name": "Linear",
              "displayName": "Linear",
              "description": "Fully Connected Layer",
              "icon": "Minus",
              "category": "linear",
              "parameters": [
                {
                  "name": "in_features",
                  "type": "number",
                  "defaultValue": 512,
                  "required": true,
                  "description": "Size of input features",
                  "min": 1
                },
                {
                  "name": "out_features",
                  "type": "number",
                  "defaultValue": 256,
                  "required": true,
                  "description": "Size of output features",
                  "min": 1
                },
                {
                  "name": "bias",
                  "type": "boolean",
                  "defaultValue": true,
                  "required": false,
                  "description": "If set to False, the layer will not learn an additive bias"
                }
              ]
            },
            "parentId": "reference_policy"
          }
        },
        {
          "node_id": "reference_policy.log_probabilities",
          "kind": "layer",
          "op": "LogSoftmax",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "logsoftmax",
              "name": "LogSoftmax",
              "displayName": "LogSoftmax",
              "description": "Log-Softmax over a dimension",
              "icon": "TrendingUp",
              "category": "activation",
              "parameters": [
                {
                  "name": "dim",
                  "type": "number",
                  "defaultValue": 1,
                  "required": true,
                  "description": "Dimension along which LogSoftmax is computed"
                }
              ]
            },
            "parentId": "reference_policy"
          }
        },
        {
          "node_id": "python@ratio#0",
          "kind": "layer",
          "op": "Python",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "python",
              "name": "Python",
              "displayName": "Python",
              "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.",
              "icon": "Hash",
              "category": "misc",
              "parameters": []
            },
            "parentId": "ratio"
          }
        },
        {
          "node_id": "python@clipped#0",
          "kind": "layer",
          "op": "Python",
          "params": {},
          "in_ports": [
            "x"
          ],
          "out_ports": [
            "y"
          ],
          "metadata": {
            "layerType": {
              "id": "python",
              "name": "Python",
              "displayName": "Python",
              "description": "A statement from the source file that has no graph equivalent. Its code is kept verbatim and its shape is unknown until the model is traced.",
              "icon": "Hash",
              "category": "misc",
              "parameters": []
            },
            "parentId": "clipped"
          }
        }
      ],
      "edges": [
        {
          "edge_id": "edge-1",
          "source_node_id": "policy.logits",
          "target_node_id": "policy.log_probabilities",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-2",
          "source_node_id": "old_policy.logits",
          "target_node_id": "old_policy.log_probabilities",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-3",
          "source_node_id": "reference_policy.logits",
          "target_node_id": "reference_policy.log_probabilities",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-4",
          "source_node_id": "input",
          "target_node_id": "policy.logits",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-5",
          "source_node_id": "input",
          "target_node_id": "policy",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-6",
          "source_node_id": "input",
          "target_node_id": "old_policy.logits",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-7",
          "source_node_id": "input",
          "target_node_id": "old_policy",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-8",
          "source_node_id": "input",
          "target_node_id": "reference_policy.logits",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-9",
          "source_node_id": "input",
          "target_node_id": "reference_policy",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-10",
          "source_node_id": "input",
          "target_node_id": "advantages",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-11",
          "source_node_id": "policy.log_probabilities",
          "target_node_id": "python@ratio#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-12",
          "source_node_id": "policy",
          "target_node_id": "ratio",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-13",
          "source_node_id": "old_policy.log_probabilities",
          "target_node_id": "python@ratio#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-14",
          "source_node_id": "old_policy",
          "target_node_id": "ratio",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-15",
          "source_node_id": "python@ratio#0",
          "target_node_id": "python@clipped#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-16",
          "source_node_id": "ratio",
          "target_node_id": "clipped",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-17",
          "source_node_id": "advantages",
          "target_node_id": "python@clipped#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-18",
          "source_node_id": "advantages",
          "target_node_id": "clipped",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-19",
          "source_node_id": "policy.log_probabilities",
          "target_node_id": "divergence",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-20",
          "source_node_id": "policy",
          "target_node_id": "divergence",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-21",
          "source_node_id": "reference_policy.log_probabilities",
          "target_node_id": "divergence",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-22",
          "source_node_id": "reference_policy",
          "target_node_id": "divergence",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-23",
          "source_node_id": "python@clipped#0",
          "target_node_id": "sub@ToyGRPO#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-24",
          "source_node_id": "clipped",
          "target_node_id": "sub@ToyGRPO#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-25",
          "source_node_id": "divergence",
          "target_node_id": "sub@ToyGRPO#0",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        },
        {
          "edge_id": "edge-26",
          "source_node_id": "sub@ToyGRPO#0",
          "target_node_id": "output",
          "source_port": "y",
          "target_port": "x",
          "metadata": {}
        }
      ],
      "inputs": [],
      "modules": [],
      "operations": [],
      "metadata": {}
    },
    "view": {
      "graph_id": "snapshot",
      "node_positions": {
        "input": {
          "x": 0,
          "y": 160
        },
        "policy": {
          "x": 340,
          "y": 160
        },
        "old_policy": {
          "x": 680,
          "y": 160
        },
        "reference_policy": {
          "x": 1020,
          "y": 160
        },
        "advantages": {
          "x": 1360,
          "y": 160
        },
        "ratio": {
          "x": 1700,
          "y": 160
        },
        "clipped": {
          "x": 2040,
          "y": 160
        },
        "divergence": {
          "x": 2380,
          "y": 160
        },
        "sub@ToyGRPO#0": {
          "x": 2720,
          "y": 160
        },
        "output": {
          "x": 3060,
          "y": 160
        },
        "policy.logits": {
          "x": 24,
          "y": 214
        },
        "policy.log_probabilities": {
          "x": 24,
          "y": 364
        },
        "old_policy.logits": {
          "x": 24,
          "y": 214
        },
        "old_policy.log_probabilities": {
          "x": 24,
          "y": 364
        },
        "reference_policy.logits": {
          "x": 24,
          "y": 214
        },
        "reference_policy.log_probabilities": {
          "x": 24,
          "y": 364
        },
        "python@ratio#0": {
          "x": 24,
          "y": 214
        },
        "python@clipped#0": {
          "x": 24,
          "y": 214
        }
      },
      "collapsed_modules": [
        "policy",
        "old_policy",
        "reference_policy",
        "advantages",
        "ratio",
        "clipped",
        "divergence"
      ],
      "selection": [],
      "layout": {}
    }
  },
  "snapshotId": "snapshot-d66f9a4e310e2cdc",
  "collaboration": {
    "walkthrough": {
      "title": "See what changes during reasoning post-training",
      "steps": [
        {
          "id": "policy",
          "title": "Keep architecture separate from the training recipe",
          "note": "This graph exposes the arithmetic of a four-choice toy policy. The original work starts from DeepSeek-V3-Base: R1-Zero applies RL directly, while R1 begins with a small cold-start SFT stage. Neither path is represented by a new attention mechanism here.",
          "view": {
            "camera": {
              "x": 308.5,
              "y": 6,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "policy": {
                "x": 340,
                "y": 160
              },
              "old_policy": {
                "x": 680,
                "y": 160
              },
              "reference_policy": {
                "x": 1020,
                "y": 160
              },
              "advantages": {
                "x": 1360,
                "y": 160
              },
              "ratio": {
                "x": 1700,
                "y": 160
              },
              "clipped": {
                "x": 2040,
                "y": 160
              },
              "divergence": {
                "x": 2380,
                "y": 160
              },
              "sub@ToyGRPO#0": {
                "x": 2720,
                "y": 160
              },
              "output": {
                "x": 3060,
                "y": 160
              },
              "policy.logits": {
                "x": 24,
                "y": 214
              },
              "policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "old_policy.logits": {
                "x": 24,
                "y": 214
              },
              "old_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "reference_policy.logits": {
                "x": 24,
                "y": 214
              },
              "reference_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "python@ratio#0": {
                "x": 24,
                "y": 214
              },
              "python@clipped#0": {
                "x": 24,
                "y": 214
              }
            },
            "expandedModuleIds": [
              "policy"
            ],
            "selectedNodeIds": [
              "policy.logits"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "policy.logits"
              },
              "snapshotId": "snapshot-d66f9a4e310e2cdc",
              "exportedAt": "2026-09-17T07:02:39.684264+00:00",
              "modelName": "ToyGRPO",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"policy.logits\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"policy.logits\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"policy.log_probabilities\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"policy.logits\",\"kind\":\"layer\",\"op\":\"Linear\",\"params\":{\"in_features\":3,\"out_features\":4},\"parent\":\"policy\"}}"
            }
          ]
        },
        {
          "id": "advantages",
          "title": "Compare a completion’s reward with its group",
          "note": "GRPO subtracts the group’s mean reward and divides by its standard deviation. The toy uses population standard deviation plus epsilon, so an equal-reward group has zero advantages. Old-policy samples provide the comparison group; no separate critic estimates the baseline.",
          "view": {
            "camera": {
              "x": -1045,
              "y": 50,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "policy": {
                "x": 340,
                "y": 160
              },
              "old_policy": {
                "x": 680,
                "y": 160
              },
              "reference_policy": {
                "x": 1020,
                "y": 160
              },
              "advantages": {
                "x": 1360,
                "y": 160
              },
              "ratio": {
                "x": 1700,
                "y": 160
              },
              "clipped": {
                "x": 2040,
                "y": 160
              },
              "divergence": {
                "x": 2380,
                "y": 160
              },
              "sub@ToyGRPO#0": {
                "x": 2720,
                "y": 160
              },
              "output": {
                "x": 3060,
                "y": 160
              },
              "policy.logits": {
                "x": 24,
                "y": 214
              },
              "policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "old_policy.logits": {
                "x": 24,
                "y": 214
              },
              "old_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "reference_policy.logits": {
                "x": 24,
                "y": 214
              },
              "reference_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "python@ratio#0": {
                "x": 24,
                "y": 214
              },
              "python@clipped#0": {
                "x": 24,
                "y": 214
              }
            },
            "expandedModuleIds": [
              "advantages"
            ],
            "selectedNodeIds": [
              "advantages"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "advantages"
              },
              "snapshotId": "snapshot-d66f9a4e310e2cdc",
              "exportedAt": "2026-09-17T07:02:39.684264+00:00",
              "modelName": "ToyGRPO",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"clipped\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@clipped#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"advantages\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"advantages\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}"
            }
          ]
        },
        {
          "id": "clip",
          "title": "Limit the surrogate incentive using old-policy ratios",
          "note": "The current-to-old probability ratio weights each advantage. Taking the minimum of unclipped and clipped surrogates handles positive and negative advantages differently. The recipe verifies both values and saturated gradients. Old-policy parameters stay fixed during the update.",
          "view": {
            "camera": {
              "x": -1770,
              "y": -57,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "policy": {
                "x": 340,
                "y": 160
              },
              "old_policy": {
                "x": 680,
                "y": 160
              },
              "reference_policy": {
                "x": 1020,
                "y": 160
              },
              "advantages": {
                "x": 1360,
                "y": 160
              },
              "ratio": {
                "x": 1700,
                "y": 160
              },
              "clipped": {
                "x": 2040,
                "y": 160
              },
              "divergence": {
                "x": 2380,
                "y": 160
              },
              "sub@ToyGRPO#0": {
                "x": 2720,
                "y": 160
              },
              "output": {
                "x": 3060,
                "y": 160
              },
              "policy.logits": {
                "x": 24,
                "y": 214
              },
              "policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "old_policy.logits": {
                "x": 24,
                "y": 214
              },
              "old_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "reference_policy.logits": {
                "x": 24,
                "y": 214
              },
              "reference_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "python@ratio#0": {
                "x": 24,
                "y": 214
              },
              "python@clipped#0": {
                "x": 24,
                "y": 214
              }
            },
            "expandedModuleIds": [
              "clipped"
            ],
            "selectedNodeIds": [
              "clipped"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "clipped"
              },
              "snapshotId": "snapshot-d66f9a4e310e2cdc",
              "exportedAt": "2026-09-17T07:02:39.684264+00:00",
              "modelName": "ToyGRPO",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"clipped\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@clipped#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"clipped\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@ToyGRPO#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@clipped#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@ToyGRPO#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"python@ratio#0\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@clipped#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"ratio\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"clipped\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[\"{\\\"id\\\":\\\"python@clipped#0\\\",\\\"kind\\\":\\\"layer\\\",\\\"op\\\":\\\"Python\\\",\\\"params\\\":{},\\\"parent\\\":\\\"clipped\\\"}\"],\"node\":{\"id\":\"clipped\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}"
            }
          ]
        },
        {
          "id": "reference",
          "title": "Retain a reference signal and distinguish later stages",
          "note": "The sampled reference penalty is exp(log πref − log π) − (log πref − log π) − 1. R1 also curates responses for a new SFT stage and runs broader final RL. Smaller distilled models receive supervised transfer from curated responses. Use the process selector to inspect these distinct paths.",
          "view": {
            "camera": {
              "x": -2065,
              "y": 50,
              "z": 1.25
            },
            "nodePositions": {
              "input": {
                "x": 0,
                "y": 160
              },
              "policy": {
                "x": 340,
                "y": 160
              },
              "old_policy": {
                "x": 680,
                "y": 160
              },
              "reference_policy": {
                "x": 1020,
                "y": 160
              },
              "advantages": {
                "x": 1360,
                "y": 160
              },
              "ratio": {
                "x": 1700,
                "y": 160
              },
              "clipped": {
                "x": 2040,
                "y": 160
              },
              "divergence": {
                "x": 2380,
                "y": 160
              },
              "sub@ToyGRPO#0": {
                "x": 2720,
                "y": 160
              },
              "output": {
                "x": 3060,
                "y": 160
              },
              "policy.logits": {
                "x": 24,
                "y": 214
              },
              "policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "old_policy.logits": {
                "x": 24,
                "y": 214
              },
              "old_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "reference_policy.logits": {
                "x": 24,
                "y": 214
              },
              "reference_policy.log_probabilities": {
                "x": 24,
                "y": 364
              },
              "python@ratio#0": {
                "x": 24,
                "y": 214
              },
              "python@clipped#0": {
                "x": 24,
                "y": 214
              }
            },
            "expandedModuleIds": [
              "divergence"
            ],
            "selectedNodeIds": [
              "divergence"
            ],
            "viewport": {
              "width": 1100,
              "height": 700
            }
          },
          "anchors": [
            {
              "target": {
                "kind": "node",
                "nodeId": "divergence"
              },
              "snapshotId": "snapshot-d66f9a4e310e2cdc",
              "exportedAt": "2026-09-17T07:02:39.684264+00:00",
              "modelName": "ToyGRPO",
              "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"divergence\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"sub@ToyGRPO#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"policy\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"divergence\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"policy.log_probabilities\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"divergence\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"reference_policy\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"divergence\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"reference_policy.log_probabilities\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"divergence\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"divergence\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}"
            }
          ]
        }
      ]
    },
    "threads": [],
    "savedViews": [
      {
        "id": "advantages",
        "title": "Compare a completion’s reward with its group",
        "note": "GRPO subtracts the group’s mean reward and divides by its standard deviation. The toy uses population standard deviation plus epsilon, so an equal-reward group has zero advantages. Old-policy samples provide the comparison group; no separate critic estimates the baseline.",
        "view": {
          "camera": {
            "x": -1045,
            "y": 50,
            "z": 1.25
          },
          "nodePositions": {
            "input": {
              "x": 0,
              "y": 160
            },
            "policy": {
              "x": 340,
              "y": 160
            },
            "old_policy": {
              "x": 680,
              "y": 160
            },
            "reference_policy": {
              "x": 1020,
              "y": 160
            },
            "advantages": {
              "x": 1360,
              "y": 160
            },
            "ratio": {
              "x": 1700,
              "y": 160
            },
            "clipped": {
              "x": 2040,
              "y": 160
            },
            "divergence": {
              "x": 2380,
              "y": 160
            },
            "sub@ToyGRPO#0": {
              "x": 2720,
              "y": 160
            },
            "output": {
              "x": 3060,
              "y": 160
            },
            "policy.logits": {
              "x": 24,
              "y": 214
            },
            "policy.log_probabilities": {
              "x": 24,
              "y": 364
            },
            "old_policy.logits": {
              "x": 24,
              "y": 214
            },
            "old_policy.log_probabilities": {
              "x": 24,
              "y": 364
            },
            "reference_policy.logits": {
              "x": 24,
              "y": 214
            },
            "reference_policy.log_probabilities": {
              "x": 24,
              "y": 364
            },
            "python@ratio#0": {
              "x": 24,
              "y": 214
            },
            "python@clipped#0": {
              "x": 24,
              "y": 214
            }
          },
          "expandedModuleIds": [
            "advantages"
          ],
          "selectedNodeIds": [
            "advantages"
          ],
          "viewport": {
            "width": 1100,
            "height": 700
          }
        },
        "anchors": [
          {
            "target": {
              "kind": "node",
              "nodeId": "advantages"
            },
            "snapshotId": "snapshot-d66f9a4e310e2cdc",
            "exportedAt": "2026-09-17T07:02:39.684264+00:00",
            "modelName": "ToyGRPO",
            "context": "{\"connections\":[\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"clipped\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"advantages\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"python@clipped#0\\\",\\\"targetPort\\\":\\\"x\\\"}\",\"{\\\"kind\\\":\\\"edge\\\",\\\"source\\\":\\\"input\\\",\\\"sourcePort\\\":\\\"y\\\",\\\"target\\\":\\\"advantages\\\",\\\"targetPort\\\":\\\"x\\\"}\"],\"contents\":[],\"node\":{\"id\":\"advantages\",\"kind\":\"module\",\"op\":\"Module\",\"params\":{}}}"
          }
        ]
      }
    ],
    "openingViewId": "advantages"
  }
}