{ "schemaVersion": 1, "slug": "sparse-experts", "title": "Let a router choose the experts", "className": "SparseRoutingComparison", "question": "How can a token use only part of a larger network?", "description": "Compare noisy top-k, Switch-style top-1 and Mixtral-style top-2. Inspect sparse dispatch, gate weights and uneven expert loads.", "category": "Modern LLMs", "tags": [ "MoE", "Switch Transformer", "Mixtral", "Sparse experts", "Routing" ], "curator": "TensorViz", "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.", "license": "No separate redistribution license has been declared for these project examples.", "limitations": "Only a reduced routing subsystem. No expert-capacity limits, token dropping, auxiliary load-balancing optimization, distributed communication or pretrained specialization. A batch can activate every expert even when each token uses just one or two. The static graph contains possible paths; recorded routes show which were used.", "openingStep": "route", "provenance": { "capturedAt": "2026-09-17T06:02:58.595933+00:00", "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c", "sourceSha256": "aa0bfb2ae5ecdfacf9f0a11e9ce4910ede59b82eeb01785880576aec856525f9", "python": "3.13.13", "torch": "2.7.1", "execution": { "inputs": { "args": [ { "shape": [ 1, 4, 8 ], "dtype": "float32", "fill": "randn" } ] }, "device": "cpu", "mode": "eval", "seed": 0, "deterministic": true }, "outcome": "passed", "outputShape": [ 3, 1, 4, 8 ], "method": "TensorViz multifile projection and isolated execution worker", "scope": "Eight-expert CPU sparse dispatch with independent dense-reference checks. Recorded seeded noisy training routes and deterministic top-1/top-2 routes; no weights or distributed runtime.", "numericalChecks": [ "Sparse token dispatch equals an independent dense weighted reference", "Top-1 retains its selected softmax probability; top-2 renormalizes the selected weights", "Only selected experts receive nonzero gradients for a single token", "Noisy top-k is reproducible with a fixed seed and noise is disabled in evaluation" ] }, "layers": [ { "id": "input", "label": "input", "op": "Input", "kind": "input", "parent": null, "params": { "shape": "1,4,8" } }, { "id": "noisy_topk", "label": "noisy_topk", "op": "Module", "kind": "module", "parent": null, "params": { "arg_0": 2, "arg_1": true, "arg_2": true }, "source": { "line": 88, "endLine": 88 } }, { "id": "switch_top1", "label": "switch_top1", "op": "Module", "kind": "module", "parent": null, "params": { "arg_0": 1, "arg_1": false, "arg_2": false }, "source": { "line": 89, "endLine": 89 } }, { "id": "mixtral_top2", "label": "mixtral_top2", "op": "Module", "kind": "module", "parent": null, "params": { "arg_0": 2, "arg_1": true, "arg_2": false }, "source": { "line": 90, "endLine": 90 } }, { "id": "stack@SparseRoutingComparison#0", "label": "stack@SparseRoutingComparison#0", "op": "torch.stack", "kind": "layer", "parent": null, "params": {}, "source": { "line": 93, "endLine": 93 } }, { "id": "output", "label": "output", "op": "Output", "kind": "output", "parent": null, "params": {} }, { "id": "noisy_topk.router", "label": "noisy_topk.router", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 2, "arg_1": true, "arg_2": false }, "source": { "line": 61, "endLine": 61 } }, { "id": "noisy_topk.expert0", "label": "noisy_topk.expert0", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 0 }, "source": { "line": 62, "endLine": 62 } }, { "id": "noisy_topk.expert1", "label": "noisy_topk.expert1", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 1 }, "source": { "line": 63, "endLine": 63 } }, { "id": "add@noisy_topk#0", "label": "add@noisy_topk#0", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert2", "label": "noisy_topk.expert2", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 2 }, "source": { "line": 64, "endLine": 64 } }, { "id": "add@noisy_topk#1", "label": "add@noisy_topk#1", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert3", "label": "noisy_topk.expert3", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 3 }, "source": { "line": 65, "endLine": 65 } }, { "id": "add@noisy_topk#2", "label": "add@noisy_topk#2", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert4", "label": "noisy_topk.expert4", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 4 }, "source": { "line": 66, "endLine": 66 } }, { "id": "add@noisy_topk#3", "label": "add@noisy_topk#3", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert5", "label": "noisy_topk.expert5", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 5 }, "source": { "line": 67, "endLine": 67 } }, { "id": "add@noisy_topk#4", "label": "add@noisy_topk#4", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert6", "label": "noisy_topk.expert6", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 6 }, "source": { "line": 68, "endLine": 68 } }, { "id": "add@noisy_topk#5", "label": "add@noisy_topk#5", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.expert7", "label": "noisy_topk.expert7", "op": "Module", "kind": "module", "parent": "noisy_topk", "params": { "arg_0": 7 }, "source": { "line": 69, "endLine": 69 } }, { "id": "add@noisy_topk#6", "label": "add@noisy_topk#6", "op": "torch.add", "kind": "layer", "parent": "noisy_topk", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "noisy_topk.router.logits", "label": "noisy_topk.router.logits", "op": "Linear", "kind": "layer", "parent": "noisy_topk.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 43, "endLine": 43 } }, { "id": "noisy_topk.router.noise_scale", "label": "noisy_topk.router.noise_scale", "op": "Linear", "kind": "layer", "parent": "noisy_topk.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 44, "endLine": 44 } }, { "id": "softplus@noisy_topk.router#0", "label": "softplus@noisy_topk.router#0", "op": "Softplus", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "mul@noisy_topk.router#0", "label": "mul@noisy_topk.router#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "add@noisy_topk.router#0", "label": "add@noisy_topk.router#0", "op": "torch.add", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "python@noisy_topk.router#0", "label": "python@noisy_topk.router#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 51, "endLine": 51 } }, { "id": "python@noisy_topk.router#1", "label": "python@noisy_topk.router#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 52, "endLine": 52 } }, { "id": "python@noisy_topk.router#2", "label": "python@noisy_topk.router#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.router", "params": {}, "source": { "line": 55, "endLine": 55 } }, { "id": "flatten@noisy_topk.expert0#0", "label": "flatten@noisy_topk.expert0#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert0#0", "label": "python@noisy_topk.expert0#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert0#1", "label": "python@noisy_topk.expert0#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert0.expert", "label": "noisy_topk.expert0.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert0#0", "label": "mul@noisy_topk.expert0#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert0#2", "label": "python@noisy_topk.expert0#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert0", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert0.expert.gate", "label": "noisy_topk.expert0.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert0.expert.up", "label": "noisy_topk.expert0.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert0.expert.silu", "label": "noisy_topk.expert0.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert0.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert0.expert#0", "label": "mul@noisy_topk.expert0.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert0.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert0.expert.down", "label": "noisy_topk.expert0.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert0.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert1#0", "label": "flatten@noisy_topk.expert1#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert1#0", "label": "python@noisy_topk.expert1#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert1#1", "label": "python@noisy_topk.expert1#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert1.expert", "label": "noisy_topk.expert1.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert1#0", "label": "mul@noisy_topk.expert1#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert1#2", "label": "python@noisy_topk.expert1#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert1", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert1.expert.gate", "label": "noisy_topk.expert1.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert1.expert.up", "label": "noisy_topk.expert1.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert1.expert.silu", "label": "noisy_topk.expert1.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert1.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert1.expert#0", "label": "mul@noisy_topk.expert1.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert1.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert1.expert.down", "label": "noisy_topk.expert1.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert1.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert2#0", "label": "flatten@noisy_topk.expert2#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert2#0", "label": "python@noisy_topk.expert2#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert2#1", "label": "python@noisy_topk.expert2#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert2.expert", "label": "noisy_topk.expert2.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert2#0", "label": "mul@noisy_topk.expert2#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert2#2", "label": "python@noisy_topk.expert2#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert2", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert2.expert.gate", "label": "noisy_topk.expert2.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert2.expert.up", "label": "noisy_topk.expert2.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert2.expert.silu", "label": "noisy_topk.expert2.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert2.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert2.expert#0", "label": "mul@noisy_topk.expert2.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert2.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert2.expert.down", "label": "noisy_topk.expert2.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert2.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert3#0", "label": "flatten@noisy_topk.expert3#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert3#0", "label": "python@noisy_topk.expert3#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert3#1", "label": "python@noisy_topk.expert3#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert3.expert", "label": "noisy_topk.expert3.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert3#0", "label": "mul@noisy_topk.expert3#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert3#2", "label": "python@noisy_topk.expert3#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert3", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert3.expert.gate", "label": "noisy_topk.expert3.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert3.expert.up", "label": "noisy_topk.expert3.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert3.expert.silu", "label": "noisy_topk.expert3.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert3.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert3.expert#0", "label": "mul@noisy_topk.expert3.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert3.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert3.expert.down", "label": "noisy_topk.expert3.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert3.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert4#0", "label": "flatten@noisy_topk.expert4#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert4#0", "label": "python@noisy_topk.expert4#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert4#1", "label": "python@noisy_topk.expert4#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert4.expert", "label": "noisy_topk.expert4.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert4#0", "label": "mul@noisy_topk.expert4#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert4#2", "label": "python@noisy_topk.expert4#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert4", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert4.expert.gate", "label": "noisy_topk.expert4.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert4.expert.up", "label": "noisy_topk.expert4.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert4.expert.silu", "label": "noisy_topk.expert4.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert4.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert4.expert#0", "label": "mul@noisy_topk.expert4.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert4.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert4.expert.down", "label": "noisy_topk.expert4.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert4.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert5#0", "label": "flatten@noisy_topk.expert5#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert5#0", "label": "python@noisy_topk.expert5#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert5#1", "label": "python@noisy_topk.expert5#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert5.expert", "label": "noisy_topk.expert5.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert5#0", "label": "mul@noisy_topk.expert5#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert5#2", "label": "python@noisy_topk.expert5#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert5", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert5.expert.gate", "label": "noisy_topk.expert5.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert5.expert.up", "label": "noisy_topk.expert5.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert5.expert.silu", "label": "noisy_topk.expert5.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert5.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert5.expert#0", "label": "mul@noisy_topk.expert5.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert5.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert5.expert.down", "label": "noisy_topk.expert5.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert5.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert6#0", "label": "flatten@noisy_topk.expert6#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert6#0", "label": "python@noisy_topk.expert6#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert6#1", "label": "python@noisy_topk.expert6#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert6.expert", "label": "noisy_topk.expert6.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert6#0", "label": "mul@noisy_topk.expert6#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert6#2", "label": "python@noisy_topk.expert6#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert6", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert6.expert.gate", "label": "noisy_topk.expert6.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert6.expert.up", "label": "noisy_topk.expert6.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert6.expert.silu", "label": "noisy_topk.expert6.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert6.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert6.expert#0", "label": "mul@noisy_topk.expert6.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert6.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert6.expert.down", "label": "noisy_topk.expert6.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert6.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@noisy_topk.expert7#0", "label": "flatten@noisy_topk.expert7#0", "op": "Flatten", "kind": "layer", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@noisy_topk.expert7#0", "label": "python@noisy_topk.expert7#0", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@noisy_topk.expert7#1", "label": "python@noisy_topk.expert7#1", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "noisy_topk.expert7.expert", "label": "noisy_topk.expert7.expert", "op": "Module", "kind": "module", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@noisy_topk.expert7#0", "label": "mul@noisy_topk.expert7#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@noisy_topk.expert7#2", "label": "python@noisy_topk.expert7#2", "op": "Python", "kind": "layer", "parent": "noisy_topk.expert7", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "noisy_topk.expert7.expert.gate", "label": "noisy_topk.expert7.expert.gate", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "noisy_topk.expert7.expert.up", "label": "noisy_topk.expert7.expert.up", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "noisy_topk.expert7.expert.silu", "label": "noisy_topk.expert7.expert.silu", "op": "SiLU", "kind": "layer", "parent": "noisy_topk.expert7.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@noisy_topk.expert7.expert#0", "label": "mul@noisy_topk.expert7.expert#0", "op": "torch.mul", "kind": "layer", "parent": "noisy_topk.expert7.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "noisy_topk.expert7.expert.down", "label": "noisy_topk.expert7.expert.down", "op": "Linear", "kind": "layer", "parent": "noisy_topk.expert7.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "switch_top1.router", "label": "switch_top1.router", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 2, "arg_1": true, "arg_2": false }, "source": { "line": 61, "endLine": 61 } }, { "id": "switch_top1.expert0", "label": "switch_top1.expert0", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 0 }, "source": { "line": 62, "endLine": 62 } }, { "id": "switch_top1.expert1", "label": "switch_top1.expert1", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 1 }, "source": { "line": 63, "endLine": 63 } }, { "id": "add@switch_top1#0", "label": "add@switch_top1#0", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert2", "label": "switch_top1.expert2", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 2 }, "source": { "line": 64, "endLine": 64 } }, { "id": "add@switch_top1#1", "label": "add@switch_top1#1", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert3", "label": "switch_top1.expert3", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 3 }, "source": { "line": 65, "endLine": 65 } }, { "id": "add@switch_top1#2", "label": "add@switch_top1#2", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert4", "label": "switch_top1.expert4", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 4 }, "source": { "line": 66, "endLine": 66 } }, { "id": "add@switch_top1#3", "label": "add@switch_top1#3", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert5", "label": "switch_top1.expert5", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 5 }, "source": { "line": 67, "endLine": 67 } }, { "id": "add@switch_top1#4", "label": "add@switch_top1#4", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert6", "label": "switch_top1.expert6", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 6 }, "source": { "line": 68, "endLine": 68 } }, { "id": "add@switch_top1#5", "label": "add@switch_top1#5", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.expert7", "label": "switch_top1.expert7", "op": "Module", "kind": "module", "parent": "switch_top1", "params": { "arg_0": 7 }, "source": { "line": 69, "endLine": 69 } }, { "id": "add@switch_top1#6", "label": "add@switch_top1#6", "op": "torch.add", "kind": "layer", "parent": "switch_top1", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "switch_top1.router.logits", "label": "switch_top1.router.logits", "op": "Linear", "kind": "layer", "parent": "switch_top1.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 43, "endLine": 43 } }, { "id": "switch_top1.router.noise_scale", "label": "switch_top1.router.noise_scale", "op": "Linear", "kind": "layer", "parent": "switch_top1.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 44, "endLine": 44 } }, { "id": "softplus@switch_top1.router#0", "label": "softplus@switch_top1.router#0", "op": "Softplus", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "mul@switch_top1.router#0", "label": "mul@switch_top1.router#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "add@switch_top1.router#0", "label": "add@switch_top1.router#0", "op": "torch.add", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "python@switch_top1.router#0", "label": "python@switch_top1.router#0", "op": "Python", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 51, "endLine": 51 } }, { "id": "python@switch_top1.router#1", "label": "python@switch_top1.router#1", "op": "Python", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 52, "endLine": 52 } }, { "id": "python@switch_top1.router#2", "label": "python@switch_top1.router#2", "op": "Python", "kind": "layer", "parent": "switch_top1.router", "params": {}, "source": { "line": 55, "endLine": 55 } }, { "id": "flatten@switch_top1.expert0#0", "label": "flatten@switch_top1.expert0#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert0#0", "label": "python@switch_top1.expert0#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert0#1", "label": "python@switch_top1.expert0#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert0.expert", "label": "switch_top1.expert0.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert0#0", "label": "mul@switch_top1.expert0#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert0#2", "label": "python@switch_top1.expert0#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert0", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert0.expert.gate", "label": "switch_top1.expert0.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert0.expert.up", "label": "switch_top1.expert0.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert0.expert.silu", "label": "switch_top1.expert0.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert0.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert0.expert#0", "label": "mul@switch_top1.expert0.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert0.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert0.expert.down", "label": "switch_top1.expert0.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert0.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert1#0", "label": "flatten@switch_top1.expert1#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert1#0", "label": "python@switch_top1.expert1#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert1#1", "label": "python@switch_top1.expert1#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert1.expert", "label": "switch_top1.expert1.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert1#0", "label": "mul@switch_top1.expert1#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert1#2", "label": "python@switch_top1.expert1#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert1", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert1.expert.gate", "label": "switch_top1.expert1.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert1.expert.up", "label": "switch_top1.expert1.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert1.expert.silu", "label": "switch_top1.expert1.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert1.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert1.expert#0", "label": "mul@switch_top1.expert1.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert1.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert1.expert.down", "label": "switch_top1.expert1.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert1.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert2#0", "label": "flatten@switch_top1.expert2#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert2#0", "label": "python@switch_top1.expert2#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert2#1", "label": "python@switch_top1.expert2#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert2.expert", "label": "switch_top1.expert2.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert2#0", "label": "mul@switch_top1.expert2#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert2#2", "label": "python@switch_top1.expert2#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert2", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert2.expert.gate", "label": "switch_top1.expert2.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert2.expert.up", "label": "switch_top1.expert2.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert2.expert.silu", "label": "switch_top1.expert2.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert2.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert2.expert#0", "label": "mul@switch_top1.expert2.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert2.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert2.expert.down", "label": "switch_top1.expert2.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert2.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert3#0", "label": "flatten@switch_top1.expert3#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert3#0", "label": "python@switch_top1.expert3#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert3#1", "label": "python@switch_top1.expert3#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert3.expert", "label": "switch_top1.expert3.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert3#0", "label": "mul@switch_top1.expert3#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert3#2", "label": "python@switch_top1.expert3#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert3", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert3.expert.gate", "label": "switch_top1.expert3.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert3.expert.up", "label": "switch_top1.expert3.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert3.expert.silu", "label": "switch_top1.expert3.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert3.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert3.expert#0", "label": "mul@switch_top1.expert3.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert3.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert3.expert.down", "label": "switch_top1.expert3.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert3.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert4#0", "label": "flatten@switch_top1.expert4#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert4#0", "label": "python@switch_top1.expert4#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert4#1", "label": "python@switch_top1.expert4#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert4.expert", "label": "switch_top1.expert4.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert4#0", "label": "mul@switch_top1.expert4#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert4#2", "label": "python@switch_top1.expert4#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert4", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert4.expert.gate", "label": "switch_top1.expert4.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert4.expert.up", "label": "switch_top1.expert4.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert4.expert.silu", "label": "switch_top1.expert4.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert4.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert4.expert#0", "label": "mul@switch_top1.expert4.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert4.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert4.expert.down", "label": "switch_top1.expert4.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert4.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert5#0", "label": "flatten@switch_top1.expert5#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert5#0", "label": "python@switch_top1.expert5#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert5#1", "label": "python@switch_top1.expert5#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert5.expert", "label": "switch_top1.expert5.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert5#0", "label": "mul@switch_top1.expert5#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert5#2", "label": "python@switch_top1.expert5#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert5", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert5.expert.gate", "label": "switch_top1.expert5.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert5.expert.up", "label": "switch_top1.expert5.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert5.expert.silu", "label": "switch_top1.expert5.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert5.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert5.expert#0", "label": "mul@switch_top1.expert5.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert5.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert5.expert.down", "label": "switch_top1.expert5.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert5.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert6#0", "label": "flatten@switch_top1.expert6#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert6#0", "label": "python@switch_top1.expert6#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert6#1", "label": "python@switch_top1.expert6#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert6.expert", "label": "switch_top1.expert6.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert6#0", "label": "mul@switch_top1.expert6#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert6#2", "label": "python@switch_top1.expert6#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert6", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert6.expert.gate", "label": "switch_top1.expert6.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert6.expert.up", "label": "switch_top1.expert6.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert6.expert.silu", "label": "switch_top1.expert6.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert6.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert6.expert#0", "label": "mul@switch_top1.expert6.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert6.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert6.expert.down", "label": "switch_top1.expert6.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert6.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@switch_top1.expert7#0", "label": "flatten@switch_top1.expert7#0", "op": "Flatten", "kind": "layer", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@switch_top1.expert7#0", "label": "python@switch_top1.expert7#0", "op": "Python", "kind": "layer", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@switch_top1.expert7#1", "label": "python@switch_top1.expert7#1", "op": "Python", "kind": "layer", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "switch_top1.expert7.expert", "label": "switch_top1.expert7.expert", "op": "Module", "kind": "module", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@switch_top1.expert7#0", "label": "mul@switch_top1.expert7#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@switch_top1.expert7#2", "label": "python@switch_top1.expert7#2", "op": "Python", "kind": "layer", "parent": "switch_top1.expert7", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "switch_top1.expert7.expert.gate", "label": "switch_top1.expert7.expert.gate", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "switch_top1.expert7.expert.up", "label": "switch_top1.expert7.expert.up", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "switch_top1.expert7.expert.silu", "label": "switch_top1.expert7.expert.silu", "op": "SiLU", "kind": "layer", "parent": "switch_top1.expert7.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@switch_top1.expert7.expert#0", "label": "mul@switch_top1.expert7.expert#0", "op": "torch.mul", "kind": "layer", "parent": "switch_top1.expert7.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "switch_top1.expert7.expert.down", "label": "switch_top1.expert7.expert.down", "op": "Linear", "kind": "layer", "parent": "switch_top1.expert7.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "mixtral_top2.router", "label": "mixtral_top2.router", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 2, "arg_1": true, "arg_2": false }, "source": { "line": 61, "endLine": 61 } }, { "id": "mixtral_top2.expert0", "label": "mixtral_top2.expert0", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 0 }, "source": { "line": 62, "endLine": 62 } }, { "id": "mixtral_top2.expert1", "label": "mixtral_top2.expert1", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 1 }, "source": { "line": 63, "endLine": 63 } }, { "id": "add@mixtral_top2#0", "label": "add@mixtral_top2#0", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert2", "label": "mixtral_top2.expert2", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 2 }, "source": { "line": 64, "endLine": 64 } }, { "id": "add@mixtral_top2#1", "label": "add@mixtral_top2#1", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert3", "label": "mixtral_top2.expert3", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 3 }, "source": { "line": 65, "endLine": 65 } }, { "id": "add@mixtral_top2#2", "label": "add@mixtral_top2#2", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert4", "label": "mixtral_top2.expert4", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 4 }, "source": { "line": 66, "endLine": 66 } }, { "id": "add@mixtral_top2#3", "label": "add@mixtral_top2#3", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert5", "label": "mixtral_top2.expert5", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 5 }, "source": { "line": 67, "endLine": 67 } }, { "id": "add@mixtral_top2#4", "label": "add@mixtral_top2#4", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert6", "label": "mixtral_top2.expert6", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 6 }, "source": { "line": 68, "endLine": 68 } }, { "id": "add@mixtral_top2#5", "label": "add@mixtral_top2#5", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.expert7", "label": "mixtral_top2.expert7", "op": "Module", "kind": "module", "parent": "mixtral_top2", "params": { "arg_0": 7 }, "source": { "line": 69, "endLine": 69 } }, { "id": "add@mixtral_top2#6", "label": "add@mixtral_top2#6", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2", "params": {}, "source": { "line": 81, "endLine": 81 } }, { "id": "mixtral_top2.router.logits", "label": "mixtral_top2.router.logits", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 43, "endLine": 43 } }, { "id": "mixtral_top2.router.noise_scale", "label": "mixtral_top2.router.noise_scale", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.router", "params": { "in_features": 8, "out_features": 8, "bias": false }, "source": { "line": 44, "endLine": 44 } }, { "id": "softplus@mixtral_top2.router#0", "label": "softplus@mixtral_top2.router#0", "op": "Softplus", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "mul@mixtral_top2.router#0", "label": "mul@mixtral_top2.router#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "add@mixtral_top2.router#0", "label": "add@mixtral_top2.router#0", "op": "torch.add", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 50, "endLine": 50 } }, { "id": "python@mixtral_top2.router#0", "label": "python@mixtral_top2.router#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 51, "endLine": 51 } }, { "id": "python@mixtral_top2.router#1", "label": "python@mixtral_top2.router#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 52, "endLine": 52 } }, { "id": "python@mixtral_top2.router#2", "label": "python@mixtral_top2.router#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.router", "params": {}, "source": { "line": 55, "endLine": 55 } }, { "id": "flatten@mixtral_top2.expert0#0", "label": "flatten@mixtral_top2.expert0#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert0#0", "label": "python@mixtral_top2.expert0#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert0#1", "label": "python@mixtral_top2.expert0#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert0.expert", "label": "mixtral_top2.expert0.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert0#0", "label": "mul@mixtral_top2.expert0#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert0#2", "label": "python@mixtral_top2.expert0#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert0", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert0.expert.gate", "label": "mixtral_top2.expert0.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert0.expert.up", "label": "mixtral_top2.expert0.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert0.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert0.expert.silu", "label": "mixtral_top2.expert0.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert0.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert0.expert#0", "label": "mul@mixtral_top2.expert0.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert0.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert0.expert.down", "label": "mixtral_top2.expert0.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert0.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert1#0", "label": "flatten@mixtral_top2.expert1#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert1#0", "label": "python@mixtral_top2.expert1#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert1#1", "label": "python@mixtral_top2.expert1#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert1.expert", "label": "mixtral_top2.expert1.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert1#0", "label": "mul@mixtral_top2.expert1#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert1#2", "label": "python@mixtral_top2.expert1#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert1", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert1.expert.gate", "label": "mixtral_top2.expert1.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert1.expert.up", "label": "mixtral_top2.expert1.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert1.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert1.expert.silu", "label": "mixtral_top2.expert1.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert1.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert1.expert#0", "label": "mul@mixtral_top2.expert1.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert1.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert1.expert.down", "label": "mixtral_top2.expert1.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert1.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert2#0", "label": "flatten@mixtral_top2.expert2#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert2#0", "label": "python@mixtral_top2.expert2#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert2#1", "label": "python@mixtral_top2.expert2#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert2.expert", "label": "mixtral_top2.expert2.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert2#0", "label": "mul@mixtral_top2.expert2#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert2#2", "label": "python@mixtral_top2.expert2#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert2", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert2.expert.gate", "label": "mixtral_top2.expert2.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert2.expert.up", "label": "mixtral_top2.expert2.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert2.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert2.expert.silu", "label": "mixtral_top2.expert2.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert2.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert2.expert#0", "label": "mul@mixtral_top2.expert2.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert2.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert2.expert.down", "label": "mixtral_top2.expert2.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert2.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert3#0", "label": "flatten@mixtral_top2.expert3#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert3#0", "label": "python@mixtral_top2.expert3#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert3#1", "label": "python@mixtral_top2.expert3#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert3.expert", "label": "mixtral_top2.expert3.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert3#0", "label": "mul@mixtral_top2.expert3#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert3#2", "label": "python@mixtral_top2.expert3#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert3", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert3.expert.gate", "label": "mixtral_top2.expert3.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert3.expert.up", "label": "mixtral_top2.expert3.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert3.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert3.expert.silu", "label": "mixtral_top2.expert3.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert3.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert3.expert#0", "label": "mul@mixtral_top2.expert3.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert3.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert3.expert.down", "label": "mixtral_top2.expert3.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert3.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert4#0", "label": "flatten@mixtral_top2.expert4#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert4#0", "label": "python@mixtral_top2.expert4#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert4#1", "label": "python@mixtral_top2.expert4#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert4.expert", "label": "mixtral_top2.expert4.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert4#0", "label": "mul@mixtral_top2.expert4#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert4#2", "label": "python@mixtral_top2.expert4#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert4", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert4.expert.gate", "label": "mixtral_top2.expert4.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert4.expert.up", "label": "mixtral_top2.expert4.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert4.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert4.expert.silu", "label": "mixtral_top2.expert4.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert4.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert4.expert#0", "label": "mul@mixtral_top2.expert4.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert4.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert4.expert.down", "label": "mixtral_top2.expert4.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert4.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert5#0", "label": "flatten@mixtral_top2.expert5#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert5#0", "label": "python@mixtral_top2.expert5#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert5#1", "label": "python@mixtral_top2.expert5#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert5.expert", "label": "mixtral_top2.expert5.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert5#0", "label": "mul@mixtral_top2.expert5#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert5#2", "label": "python@mixtral_top2.expert5#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert5", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert5.expert.gate", "label": "mixtral_top2.expert5.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert5.expert.up", "label": "mixtral_top2.expert5.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert5.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert5.expert.silu", "label": "mixtral_top2.expert5.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert5.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert5.expert#0", "label": "mul@mixtral_top2.expert5.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert5.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert5.expert.down", "label": "mixtral_top2.expert5.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert5.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert6#0", "label": "flatten@mixtral_top2.expert6#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert6#0", "label": "python@mixtral_top2.expert6#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert6#1", "label": "python@mixtral_top2.expert6#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert6.expert", "label": "mixtral_top2.expert6.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert6#0", "label": "mul@mixtral_top2.expert6#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert6#2", "label": "python@mixtral_top2.expert6#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert6", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert6.expert.gate", "label": "mixtral_top2.expert6.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert6.expert.up", "label": "mixtral_top2.expert6.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert6.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert6.expert.silu", "label": "mixtral_top2.expert6.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert6.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert6.expert#0", "label": "mul@mixtral_top2.expert6.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert6.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert6.expert.down", "label": "mixtral_top2.expert6.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert6.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } }, { "id": "flatten@mixtral_top2.expert7#0", "label": "flatten@mixtral_top2.expert7#0", "op": "Flatten", "kind": "layer", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "python@mixtral_top2.expert7#0", "label": "python@mixtral_top2.expert7#0", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "python@mixtral_top2.expert7#1", "label": "python@mixtral_top2.expert7#1", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "mixtral_top2.expert7.expert", "label": "mixtral_top2.expert7.expert", "op": "Module", "kind": "module", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 28, "endLine": 28 } }, { "id": "mul@mixtral_top2.expert7#0", "label": "mul@mixtral_top2.expert7#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 36, "endLine": 36 } }, { "id": "python@mixtral_top2.expert7#2", "label": "python@mixtral_top2.expert7#2", "op": "Python", "kind": "layer", "parent": "mixtral_top2.expert7", "params": {}, "source": { "line": 37, "endLine": 37 } }, { "id": "mixtral_top2.expert7.expert.gate", "label": "mixtral_top2.expert7.expert.gate", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 15, "endLine": 15 } }, { "id": "mixtral_top2.expert7.expert.up", "label": "mixtral_top2.expert7.expert.up", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert7.expert", "params": { "in_features": 8, "out_features": 12, "bias": false }, "source": { "line": 16, "endLine": 16 } }, { "id": "mixtral_top2.expert7.expert.silu", "label": "mixtral_top2.expert7.expert.silu", "op": "SiLU", "kind": "layer", "parent": "mixtral_top2.expert7.expert", "params": {}, "source": { "line": 17, "endLine": 17 } }, { "id": "mul@mixtral_top2.expert7.expert#0", "label": "mul@mixtral_top2.expert7.expert#0", "op": "torch.mul", "kind": "layer", "parent": "mixtral_top2.expert7.expert", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "mixtral_top2.expert7.expert.down", "label": "mixtral_top2.expert7.expert.down", "op": "Linear", "kind": "layer", "parent": "mixtral_top2.expert7.expert", "params": { "in_features": 12, "out_features": 8, "bias": false }, "source": { "line": 18, "endLine": 18 } } ], "connections": [ { "id": "edge-1", "source": "noisy_topk.router.noise_scale", "target": "softplus@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-2", "source": "softplus@noisy_topk.router#0", "target": "mul@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-3", "source": "noisy_topk.router.logits", "target": "mul@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-4", "source": "noisy_topk.router.logits", "target": "add@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-5", "source": "mul@noisy_topk.router#0", "target": "add@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-6", "source": "add@noisy_topk.router#0", "target": "python@noisy_topk.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-7", "source": "python@noisy_topk.router#0", "target": "python@noisy_topk.router#1", "sourcePort": null, "targetPort": null }, { "id": "edge-8", "source": "python@noisy_topk.router#0", "target": "python@noisy_topk.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-9", "source": "python@noisy_topk.router#1", "target": "python@noisy_topk.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-10", "source": "noisy_topk.expert0.expert.gate", "target": "noisy_topk.expert0.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-11", "source": "noisy_topk.expert0.expert.silu", "target": "mul@noisy_topk.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-12", "source": "noisy_topk.expert0.expert.up", "target": "mul@noisy_topk.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-13", "source": "mul@noisy_topk.expert0.expert#0", "target": "noisy_topk.expert0.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-14", "source": "python@noisy_topk.expert0#0", "target": "python@noisy_topk.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-15", "source": "flatten@noisy_topk.expert0#0", "target": "python@noisy_topk.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-16", "source": "python@noisy_topk.expert0#1", "target": "noisy_topk.expert0.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-17", "source": "python@noisy_topk.expert0#1", "target": "noisy_topk.expert0.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-18", "source": "python@noisy_topk.expert0#1", "target": "noisy_topk.expert0.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-19", "source": "noisy_topk.expert0.expert.down", "target": "mul@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-20", "source": "noisy_topk.expert0.expert", "target": "mul@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-21", "source": "flatten@noisy_topk.expert0#0", "target": "mul@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-22", "source": "python@noisy_topk.expert0#0", "target": "python@noisy_topk.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-23", "source": "flatten@noisy_topk.expert0#0", "target": "python@noisy_topk.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-24", "source": "mul@noisy_topk.expert0#0", "target": "python@noisy_topk.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-25", "source": "noisy_topk.expert1.expert.gate", "target": "noisy_topk.expert1.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-26", "source": "noisy_topk.expert1.expert.silu", "target": "mul@noisy_topk.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-27", "source": "noisy_topk.expert1.expert.up", "target": "mul@noisy_topk.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-28", "source": "mul@noisy_topk.expert1.expert#0", "target": "noisy_topk.expert1.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-29", "source": "python@noisy_topk.expert1#0", "target": "python@noisy_topk.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-30", "source": "flatten@noisy_topk.expert1#0", "target": "python@noisy_topk.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-31", "source": "python@noisy_topk.expert1#1", "target": "noisy_topk.expert1.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-32", "source": "python@noisy_topk.expert1#1", "target": "noisy_topk.expert1.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-33", "source": "python@noisy_topk.expert1#1", "target": "noisy_topk.expert1.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-34", "source": "noisy_topk.expert1.expert.down", "target": "mul@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-35", "source": "noisy_topk.expert1.expert", "target": "mul@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-36", "source": "flatten@noisy_topk.expert1#0", "target": "mul@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-37", "source": "python@noisy_topk.expert1#0", "target": "python@noisy_topk.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-38", "source": "flatten@noisy_topk.expert1#0", "target": "python@noisy_topk.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-39", "source": "mul@noisy_topk.expert1#0", "target": "python@noisy_topk.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-40", "source": "noisy_topk.expert2.expert.gate", "target": "noisy_topk.expert2.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-41", "source": "noisy_topk.expert2.expert.silu", "target": "mul@noisy_topk.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-42", "source": "noisy_topk.expert2.expert.up", "target": "mul@noisy_topk.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-43", "source": "mul@noisy_topk.expert2.expert#0", "target": "noisy_topk.expert2.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-44", "source": "python@noisy_topk.expert2#0", "target": "python@noisy_topk.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-45", "source": "flatten@noisy_topk.expert2#0", "target": "python@noisy_topk.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-46", "source": "python@noisy_topk.expert2#1", "target": "noisy_topk.expert2.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-47", "source": "python@noisy_topk.expert2#1", "target": "noisy_topk.expert2.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-48", "source": "python@noisy_topk.expert2#1", "target": "noisy_topk.expert2.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-49", "source": "noisy_topk.expert2.expert.down", "target": "mul@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-50", "source": "noisy_topk.expert2.expert", "target": "mul@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-51", "source": "flatten@noisy_topk.expert2#0", "target": "mul@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-52", "source": "python@noisy_topk.expert2#0", "target": "python@noisy_topk.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-53", "source": "flatten@noisy_topk.expert2#0", "target": "python@noisy_topk.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-54", "source": "mul@noisy_topk.expert2#0", "target": "python@noisy_topk.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-55", "source": "noisy_topk.expert3.expert.gate", "target": "noisy_topk.expert3.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-56", "source": "noisy_topk.expert3.expert.silu", "target": "mul@noisy_topk.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-57", "source": "noisy_topk.expert3.expert.up", "target": "mul@noisy_topk.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-58", "source": "mul@noisy_topk.expert3.expert#0", "target": "noisy_topk.expert3.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-59", "source": "python@noisy_topk.expert3#0", "target": "python@noisy_topk.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-60", "source": "flatten@noisy_topk.expert3#0", "target": "python@noisy_topk.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-61", "source": "python@noisy_topk.expert3#1", "target": "noisy_topk.expert3.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-62", "source": "python@noisy_topk.expert3#1", "target": "noisy_topk.expert3.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-63", "source": "python@noisy_topk.expert3#1", "target": "noisy_topk.expert3.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-64", "source": "noisy_topk.expert3.expert.down", "target": "mul@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-65", "source": "noisy_topk.expert3.expert", "target": "mul@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-66", "source": "flatten@noisy_topk.expert3#0", "target": "mul@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-67", "source": "python@noisy_topk.expert3#0", "target": "python@noisy_topk.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-68", "source": "flatten@noisy_topk.expert3#0", "target": "python@noisy_topk.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-69", "source": "mul@noisy_topk.expert3#0", "target": "python@noisy_topk.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-70", "source": "noisy_topk.expert4.expert.gate", "target": "noisy_topk.expert4.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-71", "source": "noisy_topk.expert4.expert.silu", "target": "mul@noisy_topk.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-72", "source": "noisy_topk.expert4.expert.up", "target": "mul@noisy_topk.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-73", "source": "mul@noisy_topk.expert4.expert#0", "target": "noisy_topk.expert4.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-74", "source": "python@noisy_topk.expert4#0", "target": "python@noisy_topk.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-75", "source": "flatten@noisy_topk.expert4#0", "target": "python@noisy_topk.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-76", "source": "python@noisy_topk.expert4#1", "target": "noisy_topk.expert4.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-77", "source": "python@noisy_topk.expert4#1", "target": "noisy_topk.expert4.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-78", "source": "python@noisy_topk.expert4#1", "target": "noisy_topk.expert4.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-79", "source": "noisy_topk.expert4.expert.down", "target": "mul@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-80", "source": "noisy_topk.expert4.expert", "target": "mul@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-81", "source": "flatten@noisy_topk.expert4#0", "target": "mul@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-82", "source": "python@noisy_topk.expert4#0", "target": "python@noisy_topk.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-83", "source": "flatten@noisy_topk.expert4#0", "target": "python@noisy_topk.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-84", "source": "mul@noisy_topk.expert4#0", "target": "python@noisy_topk.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-85", "source": "noisy_topk.expert5.expert.gate", "target": "noisy_topk.expert5.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-86", "source": "noisy_topk.expert5.expert.silu", "target": "mul@noisy_topk.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-87", "source": "noisy_topk.expert5.expert.up", "target": "mul@noisy_topk.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-88", "source": "mul@noisy_topk.expert5.expert#0", "target": "noisy_topk.expert5.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-89", "source": "python@noisy_topk.expert5#0", "target": "python@noisy_topk.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-90", "source": "flatten@noisy_topk.expert5#0", "target": "python@noisy_topk.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-91", "source": "python@noisy_topk.expert5#1", "target": "noisy_topk.expert5.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-92", "source": "python@noisy_topk.expert5#1", "target": "noisy_topk.expert5.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-93", "source": "python@noisy_topk.expert5#1", "target": "noisy_topk.expert5.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-94", "source": "noisy_topk.expert5.expert.down", "target": "mul@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-95", "source": "noisy_topk.expert5.expert", "target": "mul@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-96", "source": "flatten@noisy_topk.expert5#0", "target": "mul@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-97", "source": "python@noisy_topk.expert5#0", "target": "python@noisy_topk.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-98", "source": "flatten@noisy_topk.expert5#0", "target": "python@noisy_topk.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-99", "source": "mul@noisy_topk.expert5#0", "target": "python@noisy_topk.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-100", "source": "noisy_topk.expert6.expert.gate", "target": "noisy_topk.expert6.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-101", "source": "noisy_topk.expert6.expert.silu", "target": "mul@noisy_topk.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-102", "source": "noisy_topk.expert6.expert.up", "target": "mul@noisy_topk.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-103", "source": "mul@noisy_topk.expert6.expert#0", "target": "noisy_topk.expert6.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-104", "source": "python@noisy_topk.expert6#0", "target": "python@noisy_topk.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-105", "source": "flatten@noisy_topk.expert6#0", "target": "python@noisy_topk.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-106", "source": "python@noisy_topk.expert6#1", "target": "noisy_topk.expert6.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-107", "source": "python@noisy_topk.expert6#1", "target": "noisy_topk.expert6.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-108", "source": "python@noisy_topk.expert6#1", "target": "noisy_topk.expert6.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-109", "source": "noisy_topk.expert6.expert.down", "target": "mul@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-110", "source": "noisy_topk.expert6.expert", "target": "mul@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-111", "source": "flatten@noisy_topk.expert6#0", "target": "mul@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-112", "source": "python@noisy_topk.expert6#0", "target": "python@noisy_topk.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-113", "source": "flatten@noisy_topk.expert6#0", "target": "python@noisy_topk.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-114", "source": "mul@noisy_topk.expert6#0", "target": "python@noisy_topk.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-115", "source": "noisy_topk.expert7.expert.gate", "target": "noisy_topk.expert7.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-116", "source": "noisy_topk.expert7.expert.silu", "target": "mul@noisy_topk.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-117", "source": "noisy_topk.expert7.expert.up", "target": "mul@noisy_topk.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-118", "source": "mul@noisy_topk.expert7.expert#0", "target": "noisy_topk.expert7.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-119", "source": "python@noisy_topk.expert7#0", "target": "python@noisy_topk.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-120", "source": "flatten@noisy_topk.expert7#0", "target": "python@noisy_topk.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-121", "source": "python@noisy_topk.expert7#1", "target": "noisy_topk.expert7.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-122", "source": "python@noisy_topk.expert7#1", "target": "noisy_topk.expert7.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-123", "source": "python@noisy_topk.expert7#1", "target": "noisy_topk.expert7.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-124", "source": "noisy_topk.expert7.expert.down", "target": "mul@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-125", "source": "noisy_topk.expert7.expert", "target": "mul@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-126", "source": "flatten@noisy_topk.expert7#0", "target": "mul@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-127", "source": "python@noisy_topk.expert7#0", "target": "python@noisy_topk.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-128", "source": "flatten@noisy_topk.expert7#0", "target": "python@noisy_topk.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-129", "source": "mul@noisy_topk.expert7#0", "target": "python@noisy_topk.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-130", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-131", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-132", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-133", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-134", "source": "noisy_topk.router", "target": "noisy_topk.expert0", "sourcePort": null, "targetPort": null }, { "id": "edge-135", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-136", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-137", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-138", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-139", "source": "noisy_topk.router", "target": "noisy_topk.expert1", "sourcePort": null, "targetPort": null }, { "id": "edge-140", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-141", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-142", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-143", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-144", "source": "noisy_topk.router", "target": "noisy_topk.expert2", "sourcePort": null, "targetPort": null }, { "id": "edge-145", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-146", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-147", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-148", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-149", "source": "noisy_topk.router", "target": "noisy_topk.expert3", "sourcePort": null, "targetPort": null }, { "id": "edge-150", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-151", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-152", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-153", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-154", "source": "noisy_topk.router", "target": "noisy_topk.expert4", "sourcePort": null, "targetPort": null }, { "id": "edge-155", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-156", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-157", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-158", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-159", "source": "noisy_topk.router", "target": "noisy_topk.expert5", "sourcePort": null, "targetPort": null }, { "id": "edge-160", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-161", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-162", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-163", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-164", "source": "noisy_topk.router", "target": "noisy_topk.expert6", "sourcePort": null, "targetPort": null }, { "id": "edge-165", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-166", "source": "python@noisy_topk.router#2", "target": "flatten@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-167", "source": "python@noisy_topk.router#2", "target": "mul@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-168", "source": "python@noisy_topk.router#2", "target": "python@noisy_topk.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-169", "source": "noisy_topk.router", "target": "noisy_topk.expert7", "sourcePort": null, "targetPort": null }, { "id": "edge-170", "source": "python@noisy_topk.expert0#2", "target": "add@noisy_topk#0", "sourcePort": null, "targetPort": null }, { "id": "edge-171", "source": "noisy_topk.expert0", "target": "add@noisy_topk#0", "sourcePort": null, "targetPort": null }, { "id": "edge-172", "source": "python@noisy_topk.expert1#2", "target": "add@noisy_topk#0", "sourcePort": null, "targetPort": null }, { "id": "edge-173", "source": "noisy_topk.expert1", "target": "add@noisy_topk#0", "sourcePort": null, "targetPort": null }, { "id": "edge-174", "source": "add@noisy_topk#0", "target": "add@noisy_topk#1", "sourcePort": null, "targetPort": null }, { "id": "edge-175", "source": "python@noisy_topk.expert2#2", "target": "add@noisy_topk#1", "sourcePort": null, "targetPort": null }, { "id": "edge-176", "source": "noisy_topk.expert2", "target": "add@noisy_topk#1", "sourcePort": null, "targetPort": null }, { "id": "edge-177", "source": "add@noisy_topk#1", "target": "add@noisy_topk#2", "sourcePort": null, "targetPort": null }, { "id": "edge-178", "source": "python@noisy_topk.expert3#2", "target": "add@noisy_topk#2", "sourcePort": null, "targetPort": null }, { "id": "edge-179", "source": "noisy_topk.expert3", "target": "add@noisy_topk#2", "sourcePort": null, "targetPort": null }, { "id": "edge-180", "source": "add@noisy_topk#2", "target": "add@noisy_topk#3", "sourcePort": null, "targetPort": null }, { "id": "edge-181", "source": "python@noisy_topk.expert4#2", "target": "add@noisy_topk#3", "sourcePort": null, "targetPort": null }, { "id": "edge-182", "source": "noisy_topk.expert4", "target": "add@noisy_topk#3", "sourcePort": null, "targetPort": null }, { "id": "edge-183", "source": "add@noisy_topk#3", "target": "add@noisy_topk#4", "sourcePort": null, "targetPort": null }, { "id": "edge-184", "source": "python@noisy_topk.expert5#2", "target": "add@noisy_topk#4", "sourcePort": null, "targetPort": null }, { "id": "edge-185", "source": "noisy_topk.expert5", "target": "add@noisy_topk#4", "sourcePort": null, "targetPort": null }, { "id": "edge-186", "source": "add@noisy_topk#4", "target": "add@noisy_topk#5", "sourcePort": null, "targetPort": null }, { "id": "edge-187", "source": "python@noisy_topk.expert6#2", "target": "add@noisy_topk#5", "sourcePort": null, "targetPort": null }, { "id": "edge-188", "source": "noisy_topk.expert6", "target": "add@noisy_topk#5", "sourcePort": null, "targetPort": null }, { "id": "edge-189", "source": "add@noisy_topk#5", "target": "add@noisy_topk#6", "sourcePort": null, "targetPort": null }, { "id": "edge-190", "source": "python@noisy_topk.expert7#2", "target": "add@noisy_topk#6", "sourcePort": null, "targetPort": null }, { "id": "edge-191", "source": "noisy_topk.expert7", "target": "add@noisy_topk#6", "sourcePort": null, "targetPort": null }, { "id": "edge-192", "source": "switch_top1.router.noise_scale", "target": "softplus@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-193", "source": "softplus@switch_top1.router#0", "target": "mul@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-194", "source": "switch_top1.router.logits", "target": "mul@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-195", "source": "switch_top1.router.logits", "target": "add@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-196", "source": "mul@switch_top1.router#0", "target": "add@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-197", "source": "add@switch_top1.router#0", "target": "python@switch_top1.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-198", "source": "python@switch_top1.router#0", "target": "python@switch_top1.router#1", "sourcePort": null, "targetPort": null }, { "id": "edge-199", "source": "python@switch_top1.router#0", "target": "python@switch_top1.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-200", "source": "python@switch_top1.router#1", "target": "python@switch_top1.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-201", "source": "switch_top1.expert0.expert.gate", "target": "switch_top1.expert0.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-202", "source": "switch_top1.expert0.expert.silu", "target": "mul@switch_top1.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-203", "source": "switch_top1.expert0.expert.up", "target": "mul@switch_top1.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-204", "source": "mul@switch_top1.expert0.expert#0", "target": "switch_top1.expert0.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-205", "source": "python@switch_top1.expert0#0", "target": "python@switch_top1.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-206", "source": "flatten@switch_top1.expert0#0", "target": "python@switch_top1.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-207", "source": "python@switch_top1.expert0#1", "target": "switch_top1.expert0.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-208", "source": "python@switch_top1.expert0#1", "target": "switch_top1.expert0.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-209", "source": "python@switch_top1.expert0#1", "target": "switch_top1.expert0.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-210", "source": "switch_top1.expert0.expert.down", "target": "mul@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-211", "source": "switch_top1.expert0.expert", "target": "mul@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-212", "source": "flatten@switch_top1.expert0#0", "target": "mul@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-213", "source": "python@switch_top1.expert0#0", "target": "python@switch_top1.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-214", "source": "flatten@switch_top1.expert0#0", "target": "python@switch_top1.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-215", "source": "mul@switch_top1.expert0#0", "target": "python@switch_top1.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-216", "source": "switch_top1.expert1.expert.gate", "target": "switch_top1.expert1.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-217", "source": "switch_top1.expert1.expert.silu", "target": "mul@switch_top1.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-218", "source": "switch_top1.expert1.expert.up", "target": "mul@switch_top1.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-219", "source": "mul@switch_top1.expert1.expert#0", "target": "switch_top1.expert1.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-220", "source": "python@switch_top1.expert1#0", "target": "python@switch_top1.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-221", "source": "flatten@switch_top1.expert1#0", "target": "python@switch_top1.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-222", "source": "python@switch_top1.expert1#1", "target": "switch_top1.expert1.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-223", "source": "python@switch_top1.expert1#1", "target": "switch_top1.expert1.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-224", "source": "python@switch_top1.expert1#1", "target": "switch_top1.expert1.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-225", "source": "switch_top1.expert1.expert.down", "target": "mul@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-226", "source": "switch_top1.expert1.expert", "target": "mul@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-227", "source": "flatten@switch_top1.expert1#0", "target": "mul@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-228", "source": "python@switch_top1.expert1#0", "target": "python@switch_top1.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-229", "source": "flatten@switch_top1.expert1#0", "target": "python@switch_top1.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-230", "source": "mul@switch_top1.expert1#0", "target": "python@switch_top1.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-231", "source": "switch_top1.expert2.expert.gate", "target": "switch_top1.expert2.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-232", "source": "switch_top1.expert2.expert.silu", "target": "mul@switch_top1.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-233", "source": "switch_top1.expert2.expert.up", "target": "mul@switch_top1.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-234", "source": "mul@switch_top1.expert2.expert#0", "target": "switch_top1.expert2.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-235", "source": "python@switch_top1.expert2#0", "target": "python@switch_top1.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-236", "source": "flatten@switch_top1.expert2#0", "target": "python@switch_top1.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-237", "source": "python@switch_top1.expert2#1", "target": "switch_top1.expert2.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-238", "source": "python@switch_top1.expert2#1", "target": "switch_top1.expert2.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-239", "source": "python@switch_top1.expert2#1", "target": "switch_top1.expert2.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-240", "source": "switch_top1.expert2.expert.down", "target": "mul@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-241", "source": "switch_top1.expert2.expert", "target": "mul@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-242", "source": "flatten@switch_top1.expert2#0", "target": "mul@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-243", "source": "python@switch_top1.expert2#0", "target": "python@switch_top1.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-244", "source": "flatten@switch_top1.expert2#0", "target": "python@switch_top1.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-245", "source": "mul@switch_top1.expert2#0", "target": "python@switch_top1.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-246", "source": "switch_top1.expert3.expert.gate", "target": "switch_top1.expert3.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-247", "source": "switch_top1.expert3.expert.silu", "target": "mul@switch_top1.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-248", "source": "switch_top1.expert3.expert.up", "target": "mul@switch_top1.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-249", "source": "mul@switch_top1.expert3.expert#0", "target": "switch_top1.expert3.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-250", "source": "python@switch_top1.expert3#0", "target": "python@switch_top1.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-251", "source": "flatten@switch_top1.expert3#0", "target": "python@switch_top1.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-252", "source": "python@switch_top1.expert3#1", "target": "switch_top1.expert3.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-253", "source": "python@switch_top1.expert3#1", "target": "switch_top1.expert3.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-254", "source": "python@switch_top1.expert3#1", "target": "switch_top1.expert3.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-255", "source": "switch_top1.expert3.expert.down", "target": "mul@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-256", "source": "switch_top1.expert3.expert", "target": "mul@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-257", "source": "flatten@switch_top1.expert3#0", "target": "mul@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-258", "source": "python@switch_top1.expert3#0", "target": "python@switch_top1.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-259", "source": "flatten@switch_top1.expert3#0", "target": "python@switch_top1.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-260", "source": "mul@switch_top1.expert3#0", "target": "python@switch_top1.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-261", "source": "switch_top1.expert4.expert.gate", "target": "switch_top1.expert4.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-262", "source": "switch_top1.expert4.expert.silu", "target": "mul@switch_top1.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-263", "source": "switch_top1.expert4.expert.up", "target": "mul@switch_top1.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-264", "source": "mul@switch_top1.expert4.expert#0", "target": "switch_top1.expert4.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-265", "source": "python@switch_top1.expert4#0", "target": "python@switch_top1.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-266", "source": "flatten@switch_top1.expert4#0", "target": "python@switch_top1.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-267", "source": "python@switch_top1.expert4#1", "target": "switch_top1.expert4.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-268", "source": "python@switch_top1.expert4#1", "target": "switch_top1.expert4.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-269", "source": "python@switch_top1.expert4#1", "target": "switch_top1.expert4.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-270", "source": "switch_top1.expert4.expert.down", "target": "mul@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-271", "source": "switch_top1.expert4.expert", "target": "mul@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-272", "source": "flatten@switch_top1.expert4#0", "target": "mul@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-273", "source": "python@switch_top1.expert4#0", "target": "python@switch_top1.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-274", "source": "flatten@switch_top1.expert4#0", "target": "python@switch_top1.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-275", "source": "mul@switch_top1.expert4#0", "target": "python@switch_top1.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-276", "source": "switch_top1.expert5.expert.gate", "target": "switch_top1.expert5.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-277", "source": "switch_top1.expert5.expert.silu", "target": "mul@switch_top1.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-278", "source": "switch_top1.expert5.expert.up", "target": "mul@switch_top1.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-279", "source": "mul@switch_top1.expert5.expert#0", "target": "switch_top1.expert5.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-280", "source": "python@switch_top1.expert5#0", "target": "python@switch_top1.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-281", "source": "flatten@switch_top1.expert5#0", "target": "python@switch_top1.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-282", "source": "python@switch_top1.expert5#1", "target": "switch_top1.expert5.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-283", "source": "python@switch_top1.expert5#1", "target": "switch_top1.expert5.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-284", "source": "python@switch_top1.expert5#1", "target": "switch_top1.expert5.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-285", "source": "switch_top1.expert5.expert.down", "target": "mul@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-286", "source": "switch_top1.expert5.expert", "target": "mul@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-287", "source": "flatten@switch_top1.expert5#0", "target": "mul@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-288", "source": "python@switch_top1.expert5#0", "target": "python@switch_top1.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-289", "source": "flatten@switch_top1.expert5#0", "target": "python@switch_top1.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-290", "source": "mul@switch_top1.expert5#0", "target": "python@switch_top1.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-291", "source": "switch_top1.expert6.expert.gate", "target": "switch_top1.expert6.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-292", "source": "switch_top1.expert6.expert.silu", "target": "mul@switch_top1.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-293", "source": "switch_top1.expert6.expert.up", "target": "mul@switch_top1.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-294", "source": "mul@switch_top1.expert6.expert#0", "target": "switch_top1.expert6.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-295", "source": "python@switch_top1.expert6#0", "target": "python@switch_top1.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-296", "source": "flatten@switch_top1.expert6#0", "target": "python@switch_top1.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-297", "source": "python@switch_top1.expert6#1", "target": "switch_top1.expert6.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-298", "source": "python@switch_top1.expert6#1", "target": "switch_top1.expert6.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-299", "source": "python@switch_top1.expert6#1", "target": "switch_top1.expert6.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-300", "source": "switch_top1.expert6.expert.down", "target": "mul@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-301", "source": "switch_top1.expert6.expert", "target": "mul@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-302", "source": "flatten@switch_top1.expert6#0", "target": "mul@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-303", "source": "python@switch_top1.expert6#0", "target": "python@switch_top1.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-304", "source": "flatten@switch_top1.expert6#0", "target": "python@switch_top1.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-305", "source": "mul@switch_top1.expert6#0", "target": "python@switch_top1.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-306", "source": "switch_top1.expert7.expert.gate", "target": "switch_top1.expert7.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-307", "source": "switch_top1.expert7.expert.silu", "target": "mul@switch_top1.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-308", "source": "switch_top1.expert7.expert.up", "target": "mul@switch_top1.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-309", "source": "mul@switch_top1.expert7.expert#0", "target": "switch_top1.expert7.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-310", "source": "python@switch_top1.expert7#0", "target": "python@switch_top1.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-311", "source": "flatten@switch_top1.expert7#0", "target": "python@switch_top1.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-312", "source": "python@switch_top1.expert7#1", "target": "switch_top1.expert7.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-313", "source": "python@switch_top1.expert7#1", "target": "switch_top1.expert7.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-314", "source": "python@switch_top1.expert7#1", "target": "switch_top1.expert7.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-315", "source": "switch_top1.expert7.expert.down", "target": "mul@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-316", "source": "switch_top1.expert7.expert", "target": "mul@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-317", "source": "flatten@switch_top1.expert7#0", "target": "mul@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-318", "source": "python@switch_top1.expert7#0", "target": "python@switch_top1.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-319", "source": "flatten@switch_top1.expert7#0", "target": "python@switch_top1.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-320", "source": "mul@switch_top1.expert7#0", "target": "python@switch_top1.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-321", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-322", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-323", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-324", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-325", "source": "switch_top1.router", "target": "switch_top1.expert0", "sourcePort": null, "targetPort": null }, { "id": "edge-326", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-327", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-328", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-329", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-330", "source": "switch_top1.router", "target": "switch_top1.expert1", "sourcePort": null, "targetPort": null }, { "id": "edge-331", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-332", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-333", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-334", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-335", "source": "switch_top1.router", "target": "switch_top1.expert2", "sourcePort": null, "targetPort": null }, { "id": "edge-336", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-337", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-338", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-339", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-340", "source": "switch_top1.router", "target": "switch_top1.expert3", "sourcePort": null, "targetPort": null }, { "id": "edge-341", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-342", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-343", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-344", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-345", "source": "switch_top1.router", "target": "switch_top1.expert4", "sourcePort": null, "targetPort": null }, { "id": "edge-346", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-347", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-348", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-349", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-350", "source": "switch_top1.router", "target": "switch_top1.expert5", "sourcePort": null, "targetPort": null }, { "id": "edge-351", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-352", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-353", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-354", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-355", "source": "switch_top1.router", "target": "switch_top1.expert6", "sourcePort": null, "targetPort": null }, { "id": "edge-356", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-357", "source": "python@switch_top1.router#2", "target": "flatten@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-358", "source": "python@switch_top1.router#2", "target": "mul@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-359", "source": "python@switch_top1.router#2", "target": "python@switch_top1.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-360", "source": "switch_top1.router", "target": "switch_top1.expert7", "sourcePort": null, "targetPort": null }, { "id": "edge-361", "source": "python@switch_top1.expert0#2", "target": "add@switch_top1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-362", "source": "switch_top1.expert0", "target": "add@switch_top1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-363", "source": "python@switch_top1.expert1#2", "target": "add@switch_top1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-364", "source": "switch_top1.expert1", "target": "add@switch_top1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-365", "source": "add@switch_top1#0", "target": "add@switch_top1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-366", "source": "python@switch_top1.expert2#2", "target": "add@switch_top1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-367", "source": "switch_top1.expert2", "target": "add@switch_top1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-368", "source": "add@switch_top1#1", "target": "add@switch_top1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-369", "source": "python@switch_top1.expert3#2", "target": "add@switch_top1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-370", "source": "switch_top1.expert3", "target": "add@switch_top1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-371", "source": "add@switch_top1#2", "target": "add@switch_top1#3", "sourcePort": null, "targetPort": null }, { "id": "edge-372", "source": "python@switch_top1.expert4#2", "target": "add@switch_top1#3", "sourcePort": null, "targetPort": null }, { "id": "edge-373", "source": "switch_top1.expert4", "target": "add@switch_top1#3", "sourcePort": null, "targetPort": null }, { "id": "edge-374", "source": "add@switch_top1#3", "target": "add@switch_top1#4", "sourcePort": null, "targetPort": null }, { "id": "edge-375", "source": "python@switch_top1.expert5#2", "target": "add@switch_top1#4", "sourcePort": null, "targetPort": null }, { "id": "edge-376", "source": "switch_top1.expert5", "target": "add@switch_top1#4", "sourcePort": null, "targetPort": null }, { "id": "edge-377", "source": "add@switch_top1#4", "target": "add@switch_top1#5", "sourcePort": null, "targetPort": null }, { "id": "edge-378", "source": "python@switch_top1.expert6#2", "target": "add@switch_top1#5", "sourcePort": null, "targetPort": null }, { "id": "edge-379", "source": "switch_top1.expert6", "target": "add@switch_top1#5", "sourcePort": null, "targetPort": null }, { "id": "edge-380", "source": "add@switch_top1#5", "target": "add@switch_top1#6", "sourcePort": null, "targetPort": null }, { "id": "edge-381", "source": "python@switch_top1.expert7#2", "target": "add@switch_top1#6", "sourcePort": null, "targetPort": null }, { "id": "edge-382", "source": "switch_top1.expert7", "target": "add@switch_top1#6", "sourcePort": null, "targetPort": null }, { "id": "edge-383", "source": "mixtral_top2.router.noise_scale", "target": "softplus@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-384", "source": "softplus@mixtral_top2.router#0", "target": "mul@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-385", "source": "mixtral_top2.router.logits", "target": "mul@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-386", "source": "mixtral_top2.router.logits", "target": "add@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-387", "source": "mul@mixtral_top2.router#0", "target": "add@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-388", "source": "add@mixtral_top2.router#0", "target": "python@mixtral_top2.router#0", "sourcePort": null, "targetPort": null }, { "id": "edge-389", "source": "python@mixtral_top2.router#0", "target": "python@mixtral_top2.router#1", "sourcePort": null, "targetPort": null }, { "id": "edge-390", "source": "python@mixtral_top2.router#0", "target": "python@mixtral_top2.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-391", "source": "python@mixtral_top2.router#1", "target": "python@mixtral_top2.router#2", "sourcePort": null, "targetPort": null }, { "id": "edge-392", "source": "mixtral_top2.expert0.expert.gate", "target": "mixtral_top2.expert0.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-393", "source": "mixtral_top2.expert0.expert.silu", "target": "mul@mixtral_top2.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-394", "source": "mixtral_top2.expert0.expert.up", "target": "mul@mixtral_top2.expert0.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-395", "source": "mul@mixtral_top2.expert0.expert#0", "target": "mixtral_top2.expert0.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-396", "source": "python@mixtral_top2.expert0#0", "target": "python@mixtral_top2.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-397", "source": "flatten@mixtral_top2.expert0#0", "target": "python@mixtral_top2.expert0#1", "sourcePort": null, "targetPort": null }, { "id": "edge-398", "source": "python@mixtral_top2.expert0#1", "target": "mixtral_top2.expert0.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-399", "source": "python@mixtral_top2.expert0#1", "target": "mixtral_top2.expert0.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-400", "source": "python@mixtral_top2.expert0#1", "target": "mixtral_top2.expert0.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-401", "source": "mixtral_top2.expert0.expert.down", "target": "mul@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-402", "source": "mixtral_top2.expert0.expert", "target": "mul@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-403", "source": "flatten@mixtral_top2.expert0#0", "target": "mul@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-404", "source": "python@mixtral_top2.expert0#0", "target": "python@mixtral_top2.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-405", "source": "flatten@mixtral_top2.expert0#0", "target": "python@mixtral_top2.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-406", "source": "mul@mixtral_top2.expert0#0", "target": "python@mixtral_top2.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-407", "source": "mixtral_top2.expert1.expert.gate", "target": "mixtral_top2.expert1.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-408", "source": "mixtral_top2.expert1.expert.silu", "target": "mul@mixtral_top2.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-409", "source": "mixtral_top2.expert1.expert.up", "target": "mul@mixtral_top2.expert1.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-410", "source": "mul@mixtral_top2.expert1.expert#0", "target": "mixtral_top2.expert1.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-411", "source": "python@mixtral_top2.expert1#0", "target": "python@mixtral_top2.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-412", "source": "flatten@mixtral_top2.expert1#0", "target": "python@mixtral_top2.expert1#1", "sourcePort": null, "targetPort": null }, { "id": "edge-413", "source": "python@mixtral_top2.expert1#1", "target": "mixtral_top2.expert1.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-414", "source": "python@mixtral_top2.expert1#1", "target": "mixtral_top2.expert1.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-415", "source": "python@mixtral_top2.expert1#1", "target": "mixtral_top2.expert1.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-416", "source": "mixtral_top2.expert1.expert.down", "target": "mul@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-417", "source": "mixtral_top2.expert1.expert", "target": "mul@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-418", "source": "flatten@mixtral_top2.expert1#0", "target": "mul@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-419", "source": "python@mixtral_top2.expert1#0", "target": "python@mixtral_top2.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-420", "source": "flatten@mixtral_top2.expert1#0", "target": "python@mixtral_top2.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-421", "source": "mul@mixtral_top2.expert1#0", "target": "python@mixtral_top2.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-422", "source": "mixtral_top2.expert2.expert.gate", "target": "mixtral_top2.expert2.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-423", "source": "mixtral_top2.expert2.expert.silu", "target": "mul@mixtral_top2.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-424", "source": "mixtral_top2.expert2.expert.up", "target": "mul@mixtral_top2.expert2.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-425", "source": "mul@mixtral_top2.expert2.expert#0", "target": "mixtral_top2.expert2.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-426", "source": "python@mixtral_top2.expert2#0", "target": "python@mixtral_top2.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-427", "source": "flatten@mixtral_top2.expert2#0", "target": "python@mixtral_top2.expert2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-428", "source": "python@mixtral_top2.expert2#1", "target": "mixtral_top2.expert2.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-429", "source": "python@mixtral_top2.expert2#1", "target": "mixtral_top2.expert2.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-430", "source": "python@mixtral_top2.expert2#1", "target": "mixtral_top2.expert2.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-431", "source": "mixtral_top2.expert2.expert.down", "target": "mul@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-432", "source": "mixtral_top2.expert2.expert", "target": "mul@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-433", "source": "flatten@mixtral_top2.expert2#0", "target": "mul@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-434", "source": "python@mixtral_top2.expert2#0", "target": "python@mixtral_top2.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-435", "source": "flatten@mixtral_top2.expert2#0", "target": "python@mixtral_top2.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-436", "source": "mul@mixtral_top2.expert2#0", "target": "python@mixtral_top2.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-437", "source": "mixtral_top2.expert3.expert.gate", "target": "mixtral_top2.expert3.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-438", "source": "mixtral_top2.expert3.expert.silu", "target": "mul@mixtral_top2.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-439", "source": "mixtral_top2.expert3.expert.up", "target": "mul@mixtral_top2.expert3.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-440", "source": "mul@mixtral_top2.expert3.expert#0", "target": "mixtral_top2.expert3.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-441", "source": "python@mixtral_top2.expert3#0", "target": "python@mixtral_top2.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-442", "source": "flatten@mixtral_top2.expert3#0", "target": "python@mixtral_top2.expert3#1", "sourcePort": null, "targetPort": null }, { "id": "edge-443", "source": "python@mixtral_top2.expert3#1", "target": "mixtral_top2.expert3.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-444", "source": "python@mixtral_top2.expert3#1", "target": "mixtral_top2.expert3.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-445", "source": "python@mixtral_top2.expert3#1", "target": "mixtral_top2.expert3.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-446", "source": "mixtral_top2.expert3.expert.down", "target": "mul@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-447", "source": "mixtral_top2.expert3.expert", "target": "mul@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-448", "source": "flatten@mixtral_top2.expert3#0", "target": "mul@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-449", "source": "python@mixtral_top2.expert3#0", "target": "python@mixtral_top2.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-450", "source": "flatten@mixtral_top2.expert3#0", "target": "python@mixtral_top2.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-451", "source": "mul@mixtral_top2.expert3#0", "target": "python@mixtral_top2.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-452", "source": "mixtral_top2.expert4.expert.gate", "target": "mixtral_top2.expert4.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-453", "source": "mixtral_top2.expert4.expert.silu", "target": "mul@mixtral_top2.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-454", "source": "mixtral_top2.expert4.expert.up", "target": "mul@mixtral_top2.expert4.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-455", "source": "mul@mixtral_top2.expert4.expert#0", "target": "mixtral_top2.expert4.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-456", "source": "python@mixtral_top2.expert4#0", "target": "python@mixtral_top2.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-457", "source": "flatten@mixtral_top2.expert4#0", "target": "python@mixtral_top2.expert4#1", "sourcePort": null, "targetPort": null }, { "id": "edge-458", "source": "python@mixtral_top2.expert4#1", "target": "mixtral_top2.expert4.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-459", "source": "python@mixtral_top2.expert4#1", "target": "mixtral_top2.expert4.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-460", "source": "python@mixtral_top2.expert4#1", "target": "mixtral_top2.expert4.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-461", "source": "mixtral_top2.expert4.expert.down", "target": "mul@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-462", "source": "mixtral_top2.expert4.expert", "target": "mul@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-463", "source": "flatten@mixtral_top2.expert4#0", "target": "mul@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-464", "source": "python@mixtral_top2.expert4#0", "target": "python@mixtral_top2.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-465", "source": "flatten@mixtral_top2.expert4#0", "target": "python@mixtral_top2.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-466", "source": "mul@mixtral_top2.expert4#0", "target": "python@mixtral_top2.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-467", "source": "mixtral_top2.expert5.expert.gate", "target": "mixtral_top2.expert5.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-468", "source": "mixtral_top2.expert5.expert.silu", "target": "mul@mixtral_top2.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-469", "source": "mixtral_top2.expert5.expert.up", "target": "mul@mixtral_top2.expert5.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-470", "source": "mul@mixtral_top2.expert5.expert#0", "target": "mixtral_top2.expert5.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-471", "source": "python@mixtral_top2.expert5#0", "target": "python@mixtral_top2.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-472", "source": "flatten@mixtral_top2.expert5#0", "target": "python@mixtral_top2.expert5#1", "sourcePort": null, "targetPort": null }, { "id": "edge-473", "source": "python@mixtral_top2.expert5#1", "target": "mixtral_top2.expert5.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-474", "source": "python@mixtral_top2.expert5#1", "target": "mixtral_top2.expert5.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-475", "source": "python@mixtral_top2.expert5#1", "target": "mixtral_top2.expert5.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-476", "source": "mixtral_top2.expert5.expert.down", "target": "mul@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-477", "source": "mixtral_top2.expert5.expert", "target": "mul@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-478", "source": "flatten@mixtral_top2.expert5#0", "target": "mul@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-479", "source": "python@mixtral_top2.expert5#0", "target": "python@mixtral_top2.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-480", "source": "flatten@mixtral_top2.expert5#0", "target": "python@mixtral_top2.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-481", "source": "mul@mixtral_top2.expert5#0", "target": "python@mixtral_top2.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-482", "source": "mixtral_top2.expert6.expert.gate", "target": "mixtral_top2.expert6.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-483", "source": "mixtral_top2.expert6.expert.silu", "target": "mul@mixtral_top2.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-484", "source": "mixtral_top2.expert6.expert.up", "target": "mul@mixtral_top2.expert6.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-485", "source": "mul@mixtral_top2.expert6.expert#0", "target": "mixtral_top2.expert6.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-486", "source": "python@mixtral_top2.expert6#0", "target": "python@mixtral_top2.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-487", "source": "flatten@mixtral_top2.expert6#0", "target": "python@mixtral_top2.expert6#1", "sourcePort": null, "targetPort": null }, { "id": "edge-488", "source": "python@mixtral_top2.expert6#1", "target": "mixtral_top2.expert6.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-489", "source": "python@mixtral_top2.expert6#1", "target": "mixtral_top2.expert6.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-490", "source": "python@mixtral_top2.expert6#1", "target": "mixtral_top2.expert6.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-491", "source": "mixtral_top2.expert6.expert.down", "target": "mul@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-492", "source": "mixtral_top2.expert6.expert", "target": "mul@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-493", "source": "flatten@mixtral_top2.expert6#0", "target": "mul@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-494", "source": "python@mixtral_top2.expert6#0", "target": "python@mixtral_top2.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-495", "source": "flatten@mixtral_top2.expert6#0", "target": "python@mixtral_top2.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-496", "source": "mul@mixtral_top2.expert6#0", "target": "python@mixtral_top2.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-497", "source": "mixtral_top2.expert7.expert.gate", "target": "mixtral_top2.expert7.expert.silu", "sourcePort": null, "targetPort": null }, { "id": "edge-498", "source": "mixtral_top2.expert7.expert.silu", "target": "mul@mixtral_top2.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-499", "source": "mixtral_top2.expert7.expert.up", "target": "mul@mixtral_top2.expert7.expert#0", "sourcePort": null, "targetPort": null }, { "id": "edge-500", "source": "mul@mixtral_top2.expert7.expert#0", "target": "mixtral_top2.expert7.expert.down", "sourcePort": null, "targetPort": null }, { "id": "edge-501", "source": "python@mixtral_top2.expert7#0", "target": "python@mixtral_top2.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-502", "source": "flatten@mixtral_top2.expert7#0", "target": "python@mixtral_top2.expert7#1", "sourcePort": null, "targetPort": null }, { "id": "edge-503", "source": "python@mixtral_top2.expert7#1", "target": "mixtral_top2.expert7.expert.gate", "sourcePort": null, "targetPort": null }, { "id": "edge-504", "source": "python@mixtral_top2.expert7#1", "target": "mixtral_top2.expert7.expert.up", "sourcePort": null, "targetPort": null }, { "id": "edge-505", "source": "python@mixtral_top2.expert7#1", "target": "mixtral_top2.expert7.expert", "sourcePort": null, "targetPort": null }, { "id": "edge-506", "source": "mixtral_top2.expert7.expert.down", "target": "mul@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-507", "source": "mixtral_top2.expert7.expert", "target": "mul@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-508", "source": "flatten@mixtral_top2.expert7#0", "target": "mul@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-509", "source": "python@mixtral_top2.expert7#0", "target": "python@mixtral_top2.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-510", "source": "flatten@mixtral_top2.expert7#0", "target": "python@mixtral_top2.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-511", "source": "mul@mixtral_top2.expert7#0", "target": "python@mixtral_top2.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-512", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-513", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-514", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-515", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-516", "source": "mixtral_top2.router", "target": "mixtral_top2.expert0", "sourcePort": null, "targetPort": null }, { "id": "edge-517", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-518", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-519", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-520", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-521", "source": "mixtral_top2.router", "target": "mixtral_top2.expert1", "sourcePort": null, "targetPort": null }, { "id": "edge-522", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-523", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-524", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-525", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-526", "source": "mixtral_top2.router", "target": "mixtral_top2.expert2", "sourcePort": null, "targetPort": null }, { "id": "edge-527", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-528", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-529", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-530", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-531", "source": "mixtral_top2.router", "target": "mixtral_top2.expert3", "sourcePort": null, "targetPort": null }, { "id": "edge-532", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-533", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-534", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-535", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-536", "source": "mixtral_top2.router", "target": "mixtral_top2.expert4", "sourcePort": null, "targetPort": null }, { "id": "edge-537", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-538", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-539", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-540", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-541", "source": "mixtral_top2.router", "target": "mixtral_top2.expert5", "sourcePort": null, "targetPort": null }, { "id": "edge-542", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-543", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-544", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-545", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-546", "source": "mixtral_top2.router", "target": "mixtral_top2.expert6", "sourcePort": null, "targetPort": null }, { "id": "edge-547", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-548", "source": "python@mixtral_top2.router#2", "target": "flatten@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-549", "source": "python@mixtral_top2.router#2", "target": "mul@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-550", "source": "python@mixtral_top2.router#2", "target": "python@mixtral_top2.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-551", "source": "mixtral_top2.router", "target": "mixtral_top2.expert7", "sourcePort": null, "targetPort": null }, { "id": "edge-552", "source": "python@mixtral_top2.expert0#2", "target": "add@mixtral_top2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-553", "source": "mixtral_top2.expert0", "target": "add@mixtral_top2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-554", "source": "python@mixtral_top2.expert1#2", "target": "add@mixtral_top2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-555", "source": "mixtral_top2.expert1", "target": "add@mixtral_top2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-556", "source": "add@mixtral_top2#0", "target": "add@mixtral_top2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-557", "source": "python@mixtral_top2.expert2#2", "target": "add@mixtral_top2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-558", "source": "mixtral_top2.expert2", "target": "add@mixtral_top2#1", "sourcePort": null, "targetPort": null }, { "id": "edge-559", "source": "add@mixtral_top2#1", "target": "add@mixtral_top2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-560", "source": "python@mixtral_top2.expert3#2", "target": "add@mixtral_top2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-561", "source": "mixtral_top2.expert3", "target": "add@mixtral_top2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-562", "source": "add@mixtral_top2#2", "target": "add@mixtral_top2#3", "sourcePort": null, "targetPort": null }, { "id": "edge-563", "source": "python@mixtral_top2.expert4#2", "target": "add@mixtral_top2#3", "sourcePort": null, "targetPort": null }, { "id": "edge-564", "source": "mixtral_top2.expert4", "target": "add@mixtral_top2#3", "sourcePort": null, "targetPort": null }, { "id": "edge-565", "source": "add@mixtral_top2#3", "target": "add@mixtral_top2#4", "sourcePort": null, "targetPort": null }, { "id": "edge-566", "source": "python@mixtral_top2.expert5#2", "target": "add@mixtral_top2#4", "sourcePort": null, "targetPort": null }, { "id": "edge-567", "source": "mixtral_top2.expert5", "target": "add@mixtral_top2#4", "sourcePort": null, "targetPort": null }, { "id": "edge-568", "source": "add@mixtral_top2#4", "target": "add@mixtral_top2#5", "sourcePort": null, "targetPort": null }, { "id": "edge-569", "source": "python@mixtral_top2.expert6#2", "target": "add@mixtral_top2#5", "sourcePort": null, "targetPort": null }, { "id": "edge-570", "source": "mixtral_top2.expert6", "target": "add@mixtral_top2#5", "sourcePort": null, "targetPort": null }, { "id": "edge-571", "source": "add@mixtral_top2#5", "target": "add@mixtral_top2#6", "sourcePort": null, "targetPort": null }, { "id": "edge-572", "source": "python@mixtral_top2.expert7#2", "target": "add@mixtral_top2#6", "sourcePort": null, "targetPort": null }, { "id": "edge-573", "source": "mixtral_top2.expert7", "target": "add@mixtral_top2#6", "sourcePort": null, "targetPort": null }, { "id": "edge-574", "source": "input", "target": "noisy_topk.router.logits", "sourcePort": null, "targetPort": null }, { "id": "edge-575", "source": "input", "target": "noisy_topk.router.noise_scale", "sourcePort": null, "targetPort": null }, { "id": "edge-576", "source": "input", "target": "python@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-577", "source": "input", "target": "flatten@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-578", "source": "input", "target": "mul@noisy_topk.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-579", "source": "input", "target": "python@noisy_topk.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-580", "source": "input", "target": "python@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-581", "source": "input", "target": "flatten@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-582", "source": "input", "target": "mul@noisy_topk.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-583", "source": "input", "target": "python@noisy_topk.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-584", "source": "input", "target": "python@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-585", "source": "input", "target": "flatten@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-586", "source": "input", "target": "mul@noisy_topk.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-587", "source": "input", "target": "python@noisy_topk.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-588", "source": "input", "target": "python@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-589", "source": "input", "target": "flatten@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-590", "source": "input", "target": "mul@noisy_topk.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-591", "source": "input", "target": "python@noisy_topk.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-592", "source": "input", "target": "python@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-593", "source": "input", "target": "flatten@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-594", "source": "input", "target": "mul@noisy_topk.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-595", "source": "input", "target": "python@noisy_topk.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-596", "source": "input", "target": "python@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-597", "source": "input", "target": "flatten@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-598", "source": "input", "target": "mul@noisy_topk.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-599", "source": "input", "target": "python@noisy_topk.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-600", "source": "input", "target": "python@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-601", "source": "input", "target": "flatten@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-602", "source": "input", "target": "mul@noisy_topk.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-603", "source": "input", "target": "python@noisy_topk.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-604", "source": "input", "target": "python@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-605", "source": "input", "target": "flatten@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-606", "source": "input", "target": "mul@noisy_topk.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-607", "source": "input", "target": "python@noisy_topk.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-608", "source": "input", "target": "noisy_topk", "sourcePort": null, "targetPort": null }, { "id": "edge-609", "source": "input", "target": "switch_top1.router.logits", "sourcePort": null, "targetPort": null }, { "id": "edge-610", "source": "input", "target": "switch_top1.router.noise_scale", "sourcePort": null, "targetPort": null }, { "id": "edge-611", "source": "input", "target": "python@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-612", "source": "input", "target": "flatten@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-613", "source": "input", "target": "mul@switch_top1.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-614", "source": "input", "target": "python@switch_top1.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-615", "source": "input", "target": "python@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-616", "source": "input", "target": "flatten@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-617", "source": "input", "target": "mul@switch_top1.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-618", "source": "input", "target": "python@switch_top1.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-619", "source": "input", "target": "python@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-620", "source": "input", "target": "flatten@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-621", "source": "input", "target": "mul@switch_top1.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-622", "source": "input", "target": "python@switch_top1.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-623", "source": "input", "target": "python@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-624", "source": "input", "target": "flatten@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-625", "source": "input", "target": "mul@switch_top1.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-626", "source": "input", "target": "python@switch_top1.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-627", "source": "input", "target": "python@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-628", "source": "input", "target": "flatten@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-629", "source": "input", "target": "mul@switch_top1.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-630", "source": "input", "target": "python@switch_top1.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-631", "source": "input", "target": "python@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-632", "source": "input", "target": "flatten@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-633", "source": "input", "target": "mul@switch_top1.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-634", "source": "input", "target": "python@switch_top1.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-635", "source": "input", "target": "python@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-636", "source": "input", "target": "flatten@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-637", "source": "input", "target": "mul@switch_top1.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-638", "source": "input", "target": "python@switch_top1.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-639", "source": "input", "target": "python@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-640", "source": "input", "target": "flatten@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-641", "source": "input", "target": "mul@switch_top1.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-642", "source": "input", "target": "python@switch_top1.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-643", "source": "input", "target": "switch_top1", "sourcePort": null, "targetPort": null }, { "id": "edge-644", "source": "input", "target": "mixtral_top2.router.logits", "sourcePort": null, "targetPort": null }, { "id": "edge-645", "source": "input", "target": "mixtral_top2.router.noise_scale", "sourcePort": null, "targetPort": null }, { "id": "edge-646", "source": "input", "target": "python@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-647", "source": "input", "target": "flatten@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-648", "source": "input", "target": "mul@mixtral_top2.expert0#0", "sourcePort": null, "targetPort": null }, { "id": "edge-649", "source": "input", "target": "python@mixtral_top2.expert0#2", "sourcePort": null, "targetPort": null }, { "id": "edge-650", "source": "input", "target": "python@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-651", "source": "input", "target": "flatten@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-652", "source": "input", "target": "mul@mixtral_top2.expert1#0", "sourcePort": null, "targetPort": null }, { "id": "edge-653", "source": "input", "target": "python@mixtral_top2.expert1#2", "sourcePort": null, "targetPort": null }, { "id": "edge-654", "source": "input", "target": "python@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-655", "source": "input", "target": "flatten@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-656", "source": "input", "target": "mul@mixtral_top2.expert2#0", "sourcePort": null, "targetPort": null }, { "id": "edge-657", "source": "input", "target": "python@mixtral_top2.expert2#2", "sourcePort": null, "targetPort": null }, { "id": "edge-658", "source": "input", "target": "python@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-659", "source": "input", "target": "flatten@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-660", "source": "input", "target": "mul@mixtral_top2.expert3#0", "sourcePort": null, "targetPort": null }, { "id": "edge-661", "source": "input", "target": "python@mixtral_top2.expert3#2", "sourcePort": null, "targetPort": null }, { "id": "edge-662", "source": "input", "target": "python@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-663", "source": "input", "target": "flatten@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-664", "source": "input", "target": "mul@mixtral_top2.expert4#0", "sourcePort": null, "targetPort": null }, { "id": "edge-665", "source": "input", "target": "python@mixtral_top2.expert4#2", "sourcePort": null, "targetPort": null }, { "id": "edge-666", "source": "input", "target": "python@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-667", "source": "input", "target": "flatten@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-668", "source": "input", "target": "mul@mixtral_top2.expert5#0", "sourcePort": null, "targetPort": null }, { "id": "edge-669", "source": "input", "target": "python@mixtral_top2.expert5#2", "sourcePort": null, "targetPort": null }, { "id": "edge-670", "source": "input", "target": "python@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-671", "source": "input", "target": "flatten@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-672", "source": "input", "target": "mul@mixtral_top2.expert6#0", "sourcePort": null, "targetPort": null }, { "id": "edge-673", "source": "input", "target": "python@mixtral_top2.expert6#2", "sourcePort": null, "targetPort": null }, { "id": "edge-674", "source": "input", "target": "python@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-675", "source": "input", "target": "flatten@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-676", "source": "input", "target": "mul@mixtral_top2.expert7#0", "sourcePort": null, "targetPort": null }, { "id": "edge-677", "source": "input", "target": "python@mixtral_top2.expert7#2", "sourcePort": null, "targetPort": null }, { "id": "edge-678", "source": "input", "target": "mixtral_top2", "sourcePort": null, "targetPort": null }, { "id": "edge-679", "source": "add@noisy_topk#6", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-680", "source": "noisy_topk", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-681", "source": "add@switch_top1#6", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-682", "source": "switch_top1", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-683", "source": "add@mixtral_top2#6", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-684", "source": "mixtral_top2", "target": "stack@SparseRoutingComparison#0", "sourcePort": null, "targetPort": null }, { "id": "edge-685", "source": "stack@SparseRoutingComparison#0", "target": "output", "sourcePort": null, "targetPort": null } ], "steps": [ { "id": "route", "target": "mixtral_top2.router", "title": "Score and select two experts", "note": "The router scores eight experts, selects the two largest probabilities and renormalizes those selected weights. Each token can choose a different pair. The recorded diagram shows actual routes for synthetic inputs." }, { "id": "dispatch", "target": "mixtral_top2.expert0", "title": "Send only selected tokens to an expert", "note": "Dispatch gathers tokens with a nonzero gate for expert 0, evaluates its SwiGLU network, multiplies by the gate and scatters the result back. Unselected tokens never enter this expert's linear layers." }, { "id": "switch", "target": "switch_top1.router", "title": "Compare Switch-style top-1", "note": "Switch sends each token to one expert and retains the winning full-softmax probability as its output weight. This differs from normalizing a top-1 gate to one. The simplified block omits capacity limits and auxiliary-loss training." }, { "id": "noise", "target": "noisy_topk.router", "title": "Explore the earlier noisy top-k route", "note": "The 2017 mechanism adds Gaussian noise scaled by softplus of a learned projection during training, then selects top-k. Evaluation disables noise. A seeded training case below lets you inspect the resulting changed routes." } ], "milestone": { "name": "Sparse MoE / Switch / Mixtral", "year": 2017, "kind": "Component", "fidelity": "Routing comparison with eight width-8, hidden-12 SwiGLU experts per branch. Captures 2017-style noisy top-k, Switch-style probability-weighted top-1 and Mixtral-style normalized top-2. Original models use different expert and training configurations.", "papers": [ { "title": "Outrageously Large Neural Networks: The Sparsely-Gated Mixture-of-Experts Layer (2017)", "url": "https://arxiv.org/abs/1701.06538" }, { "title": "Switch Transformers (2021)", "url": "https://arxiv.org/abs/2101.03961" }, { "title": "Mixtral of Experts (2024 paper)", "url": "https://arxiv.org/abs/2401.04088" } ] }, "experiment": { "kind": "routing", "title": "Choose experts for each token.", "description": "Recorded sparse dispatch through eight experts. Parameter counts include expert matrices only, excluding the router. More total parameters does not mean all are evaluated for every token. Batch imbalance remains visible in the load counts.", "controlLabel": "Routing rule and input", "cases": [ { "id": "switch", "label": "Switch-style top-1", "target": "switch_top1", "note": "Eight untrained SwiGLU experts share identical weights across these cases. Lines show actual selected routes, not learned semantic specializations. Switch keeps the winning softmax probability; the top-2 variants normalize over selected experts. The 2017-style case adds seeded learned-scale Gaussian noise during training. No capacity limits or dropped tokens are modeled.", "routing": { "sourceLabel": "Tokens", "destinationLabel": "Experts", "sources": [ "Token 0", "Token 1", "Token 2", "Token 3" ], "destinations": [ "E0", "E1", "E2", "E3", "E4", "E5", "E6", "E7" ], "weights": [ [ 0.8623061727875724, 0, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0.8623061727875724, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0.8623061727875724, 0, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0.8623061727875725, 0 ] ] }, "vectors": [ { "label": "Pre-noise probabilities · token 0", "values": [ 0.8623061727875724, 0.042931696378605964, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596 ] }, { "label": "Tokens processed · each expert", "values": [ 1, 0, 1, 0, 1, 0, 1, 0 ] }, { "label": "Combined output · token 0", "values": [ -0.013819081787737272, -0.008929865297250941, 0.006910783950808611, 0.07879998496959037, -0.08213379326969798, 0.03648980355545727, 0.0040303730438244366, -0.09668960041203195 ] } ], "metrics": [ { "label": "Active expert parameters per token", "value": 288 }, { "label": "Total expert parameters", "value": 2304 }, { "label": "Experts used across this batch", "value": 4 }, { "label": "Maximum expert token load", "value": 1 } ] }, { "id": "mixtral", "label": "Mixtral-style top-2", "target": "mixtral_top2", "note": "Eight untrained SwiGLU experts share identical weights across these cases. Lines show actual selected routes, not learned semantic specializations. Switch keeps the winning softmax probability; the top-2 variants normalize over selected experts. The 2017-style case adds seeded learned-scale Gaussian noise during training. No capacity limits or dropped tokens are modeled.", "routing": { "sourceLabel": "Tokens", "destinationLabel": "Experts", "sources": [ "Token 0", "Token 1", "Token 2", "Token 3" ], "destinations": [ "E0", "E1", "E2", "E3", "E4", "E5", "E6", "E7" ], "weights": [ [ 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0, 0, 0 ], [ 0, 0, 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0.9525741268224333, 0.04742587317756678, 0, 0 ], [ 0, 0, 0, 0, 0, 0, 0.9525741268224333, 0.04742587317756678 ] ] }, "vectors": [ { "label": "Pre-noise probabilities · token 0", "values": [ 0.8623061727875724, 0.042931696378605964, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596 ] }, { "label": "Tokens processed · each expert", "values": [ 1, 1, 1, 1, 1, 1, 1, 1 ] }, { "label": "Combined output · token 0", "values": [ -0.01028808701351071, -0.012539576006071402, 0.005034020422205923, 0.08949439366357365, -0.09354091568866689, 0.03533759136369533, 0.0023051505910873264, -0.11025425849516826 ] } ], "metrics": [ { "label": "Active expert parameters per token", "value": 576 }, { "label": "Total expert parameters", "value": 2304 }, { "label": "Experts used across this batch", "value": 8 }, { "label": "Maximum expert token load", "value": 1 } ] }, { "id": "noisy", "label": "2017-style noisy top-2 · training", "target": "noisy_topk", "note": "Eight untrained SwiGLU experts share identical weights across these cases. Lines show actual selected routes, not learned semantic specializations. Switch keeps the winning softmax probability; the top-2 variants normalize over selected experts. The 2017-style case adds seeded learned-scale Gaussian noise during training. No capacity limits or dropped tokens are modeled.", "routing": { "sourceLabel": "Tokens", "destinationLabel": "Experts", "sources": [ "Token 0", "Token 1", "Token 2", "Token 3" ], "destinations": [ "E0", "E1", "E2", "E3", "E4", "E5", "E6", "E7" ], "weights": [ [ 0.9234378603514575, 0.07656213964854247, 0, 0, 0, 0, 0, 0 ], [ 0.038824438459841154, 0, 0.9611755615401588, 0, 0, 0, 0, 0 ], [ 0, 0, 0, 0, 0.8456262887762321, 0, 0.15437371122376797, 0 ], [ 0, 0, 0, 0, 0, 0, 0.9435412154840493, 0.056458784515950634 ] ] }, "vectors": [ { "label": "Pre-noise probabilities · token 0", "values": [ 0.8623061727875724, 0.042931696378605964, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596 ] }, { "label": "Tokens processed · each expert", "values": [ 2, 1, 1, 0, 1, 0, 2, 1 ] }, { "label": "Combined output · token 0", "values": [ -0.006763147044278028, -0.013881190681527805, 0.0032030717136559772, 0.0883342208982568, -0.09249155282232738, 0.031050054786154167, 0.0008498716802964276, -0.10910245289976299 ] } ], "metrics": [ { "label": "Active expert parameters per token", "value": 576 }, { "label": "Total expert parameters", "value": 2304 }, { "label": "Experts used across this batch", "value": 6 }, { "label": "Maximum expert token load", "value": 2 } ] }, { "id": "skewed", "label": "Top-2 with all tokens choosing the same experts", "target": "mixtral_top2", "note": "Eight untrained SwiGLU experts share identical weights across these cases. Lines show actual selected routes, not learned semantic specializations. Switch keeps the winning softmax probability; the top-2 variants normalize over selected experts. The 2017-style case adds seeded learned-scale Gaussian noise during training. No capacity limits or dropped tokens are modeled.", "routing": { "sourceLabel": "Tokens", "destinationLabel": "Experts", "sources": [ "Token 0", "Token 1", "Token 2", "Token 3" ], "destinations": [ "E0", "E1", "E2", "E3", "E4", "E5", "E6", "E7" ], "weights": [ [ 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0, 0, 0 ], [ 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0, 0, 0 ], [ 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0, 0, 0 ], [ 0.9525741268224333, 0.04742587317756678, 0, 0, 0, 0, 0, 0 ] ] }, "vectors": [ { "label": "Pre-noise probabilities · token 0", "values": [ 0.8623061727875724, 0.042931696378605964, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596, 0.015793688472303596 ] }, { "label": "Tokens processed · each expert", "values": [ 4, 4, 0, 0, 0, 0, 0, 0 ] }, { "label": "Combined output · token 0", "values": [ -0.010288087013510708, -0.012539576006071402, 0.005034020422205925, 0.08949439366357365, -0.09354091568866689, 0.03533759136369533, 0.002305150591087327, -0.11025425849516826 ] } ], "metrics": [ { "label": "Active expert parameters per token", "value": 576 }, { "label": "Total expert parameters", "value": 2304 }, { "label": "Experts used across this batch", "value": 2 }, { "label": "Maximum expert token load", "value": 4 } ] } ] }, "revision": "5e72fa1398a92512", "files": { "graph.tensorviz.json": { "url": "/models/sparse-experts/5e72fa1398a92512/graph.tensorviz.json", "sha256": "43e599238c2110537a4a4408b0656a4868bfe375e493d97d3808289ceb7c0994", "bytes": 857343 }, "model.py": { "url": "/models/sparse-experts/5e72fa1398a92512/model.py", "sha256": "aa0bfb2ae5ecdfacf9f0a11e9ce4910ede59b82eeb01785880576aec856525f9", "bytes": 8092 }, "run.py": { "url": "/models/sparse-experts/5e72fa1398a92512/run.py", "sha256": "87baa04300b3bcfaaf430433de3196c863d6418e845b5fa388a060f1eeab7d62", "bytes": 672 }, "requirements.txt": { "url": "/models/sparse-experts/5e72fa1398a92512/requirements.txt", "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf", "bytes": 91 }, "preview.svg": { "url": "/models/sparse-experts/5e72fa1398a92512/preview.svg", "sha256": "cddbac1b79d67cf93cab022400ddae0a162e3392d4ff900651569c73e6aae5e5", "bytes": 1962 } } }