{
  "schemaVersion": 1,
  "slug": "bert",
  "title": "Learn from both sides of a masked token",
  "className": "TinyBertPretraining",
  "question": "What changes when the encoder can see both left and right context?",
  "description": "Explore a bidirectional BERT-style encoder, then inspect which token targets and sentence-pair labels contribute to its pretraining objectives.",
  "category": "Language milestones",
  "tags": [
    "BERT",
    "Bidirectional attention",
    "Masked language modeling",
    "NSP",
    "Pretraining"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "Untrained fixed-length model without padding or a tokenizer; dropout is omitted. The numerical cases always mask chosen tokens to isolate the MLM objective, rather than reproduce the complete 15% and 80/10/10 corruption sampler. NSP uses synthetic labels. The recipe packs two output heads into one 98-value tensor for verification.",
  "openingStep": "bidirectional",
  "provenance": {
    "capturedAt": "2026-09-17T06:32:45.051926+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "fa041d05ccb2a15cb1e139d53180ab5ca4930dd11ba12f3fd3aec170228f0fc5",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              6
            ],
            "dtype": "int64",
            "fill": "ones"
          },
          {
            "shape": [
              1,
              6
            ],
            "dtype": "int64",
            "fill": "ones"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      98
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "One reduced BERT pretraining forward with MLM/NSP heads, bidirectional influence, ignored-target loss behavior and tied-weight gradient checks. Output packs 96 MLM logits and two NSP logits.",
    "numericalChecks": [
      "A later token can change earlier token logits through bidirectional attention",
      "MLM cross entropy ignores all unselected token positions",
      "The MLM decoder shares the token-embedding weight and receives a learning signal",
      "Packed MLM/NSP logits have the declared shapes and sentence-pair probabilities sum to one"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,5,16"
      }
    },
    {
      "id": "python@TinyBertPretraining#0",
      "label": "python@TinyBertPretraining#0",
      "op": "Python",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 64,
        "endLine": 64
      }
    },
    {
      "id": "word_embeddings",
      "label": "word_embeddings",
      "op": "Embedding",
      "kind": "layer",
      "parent": null,
      "params": {
        "num_embeddings": 16,
        "embedding_dim": 8
      },
      "source": {
        "line": 49,
        "endLine": 49
      }
    },
    {
      "id": "position_embeddings",
      "label": "position_embeddings",
      "op": "Embedding",
      "kind": "layer",
      "parent": null,
      "params": {
        "num_embeddings": 6,
        "embedding_dim": 8
      },
      "source": {
        "line": 50,
        "endLine": 50
      }
    },
    {
      "id": "add@TinyBertPretraining#0",
      "label": "add@TinyBertPretraining#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 65,
        "endLine": 69
      }
    },
    {
      "id": "segment_embeddings",
      "label": "segment_embeddings",
      "op": "Embedding",
      "kind": "layer",
      "parent": null,
      "params": {
        "num_embeddings": 2,
        "embedding_dim": 8
      },
      "source": {
        "line": 51,
        "endLine": 51
      }
    },
    {
      "id": "add@TinyBertPretraining#1",
      "label": "add@TinyBertPretraining#1",
      "op": "torch.add",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 65,
        "endLine": 69
      }
    },
    {
      "id": "embedding_norm",
      "label": "embedding_norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": null,
      "params": {
        "normalized_shape": 8
      },
      "source": {
        "line": 52,
        "endLine": 52
      }
    },
    {
      "id": "encoder",
      "label": "encoder",
      "op": "Module",
      "kind": "module",
      "parent": null,
      "params": {},
      "source": {
        "line": 53,
        "endLine": 53
      }
    },
    {
      "id": "mlm_transform",
      "label": "mlm_transform",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 54,
        "endLine": 54
      }
    },
    {
      "id": "mlm_gelu",
      "label": "mlm_gelu",
      "op": "GELU",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 55,
        "endLine": 55
      }
    },
    {
      "id": "mlm_norm",
      "label": "mlm_norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": null,
      "params": {
        "normalized_shape": 8
      },
      "source": {
        "line": 56,
        "endLine": 56
      }
    },
    {
      "id": "mlm_decoder",
      "label": "mlm_decoder",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 16
      },
      "source": {
        "line": 57,
        "endLine": 57
      }
    },
    {
      "id": "flatten@TinyBertPretraining#0",
      "label": "flatten@TinyBertPretraining#0",
      "op": "Flatten",
      "kind": "layer",
      "parent": null,
      "params": {
        "start_dim": 1
      },
      "source": {
        "line": 73,
        "endLine": 73
      }
    },
    {
      "id": "pooler",
      "label": "pooler",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 59,
        "endLine": 59
      }
    },
    {
      "id": "pooler_tanh",
      "label": "pooler_tanh",
      "op": "Tanh",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 60,
        "endLine": 60
      }
    },
    {
      "id": "nsp_head",
      "label": "nsp_head",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 2
      },
      "source": {
        "line": 61,
        "endLine": 61
      }
    },
    {
      "id": "cat@TinyBertPretraining#0",
      "label": "cat@TinyBertPretraining#0",
      "op": "torch.cat",
      "kind": "layer",
      "parent": null,
      "params": {
        "dim": 1
      },
      "source": {
        "line": 73,
        "endLine": 73
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    },
    {
      "id": "encoder.attention",
      "label": "encoder.attention",
      "op": "Module",
      "kind": "module",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 34,
        "endLine": 34
      }
    },
    {
      "id": "add@encoder#0",
      "label": "add@encoder#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 42,
        "endLine": 42
      }
    },
    {
      "id": "encoder.attention_norm",
      "label": "encoder.attention_norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "normalized_shape": 8
      },
      "source": {
        "line": 35,
        "endLine": 35
      }
    },
    {
      "id": "encoder.up",
      "label": "encoder.up",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "in_features": 8,
        "out_features": 32
      },
      "source": {
        "line": 36,
        "endLine": 36
      }
    },
    {
      "id": "encoder.gelu",
      "label": "encoder.gelu",
      "op": "GELU",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 37,
        "endLine": 37
      }
    },
    {
      "id": "encoder.down",
      "label": "encoder.down",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "in_features": 32,
        "out_features": 8
      },
      "source": {
        "line": 38,
        "endLine": 38
      }
    },
    {
      "id": "add@encoder#1",
      "label": "add@encoder#1",
      "op": "torch.add",
      "kind": "layer",
      "parent": "encoder",
      "params": {},
      "source": {
        "line": 43,
        "endLine": 43
      }
    },
    {
      "id": "encoder.output_norm",
      "label": "encoder.output_norm",
      "op": "LayerNorm",
      "kind": "layer",
      "parent": "encoder",
      "params": {
        "normalized_shape": 8
      },
      "source": {
        "line": 39,
        "endLine": 39
      }
    },
    {
      "id": "encoder.attention.query",
      "label": "encoder.attention.query",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 16,
        "endLine": 16
      }
    },
    {
      "id": "python@encoder.attention#0",
      "label": "python@encoder.attention#0",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "encoder.attention.key",
      "label": "encoder.attention.key",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 17,
        "endLine": 17
      }
    },
    {
      "id": "python@encoder.attention#1",
      "label": "python@encoder.attention#1",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 25,
        "endLine": 25
      }
    },
    {
      "id": "div@encoder.attention#0",
      "label": "div@encoder.attention#0",
      "op": "torch.div",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 27,
        "endLine": 27
      }
    },
    {
      "id": "encoder.attention.value",
      "label": "encoder.attention.value",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 18,
        "endLine": 18
      }
    },
    {
      "id": "python@encoder.attention#2",
      "label": "python@encoder.attention#2",
      "op": "Python",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 26,
        "endLine": 26
      }
    },
    {
      "id": "encoder.attention.softmax",
      "label": "encoder.attention.softmax",
      "op": "Softmax",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {},
      "source": {
        "line": 19,
        "endLine": 19
      }
    },
    {
      "id": "encoder.attention.project",
      "label": "encoder.attention.project",
      "op": "Linear",
      "kind": "layer",
      "parent": "encoder.attention",
      "params": {
        "in_features": 8,
        "out_features": 8
      },
      "source": {
        "line": 20,
        "endLine": 20
      }
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "encoder.attention.query",
      "target": "python@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "encoder.attention.key",
      "target": "python@encoder.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "encoder.attention.value",
      "target": "python@encoder.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "python@encoder.attention#0",
      "target": "div@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "python@encoder.attention#1",
      "target": "div@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "div@encoder.attention#0",
      "target": "encoder.attention.softmax",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-7",
      "source": "encoder.attention.softmax",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-8",
      "source": "python@encoder.attention#2",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-9",
      "source": "encoder.attention.project",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-10",
      "source": "encoder.attention",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-11",
      "source": "add@encoder#0",
      "target": "encoder.attention_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-12",
      "source": "encoder.attention_norm",
      "target": "encoder.up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-13",
      "source": "encoder.up",
      "target": "encoder.gelu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-14",
      "source": "encoder.gelu",
      "target": "encoder.down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-15",
      "source": "encoder.attention_norm",
      "target": "add@encoder#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-16",
      "source": "encoder.down",
      "target": "add@encoder#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-17",
      "source": "add@encoder#1",
      "target": "encoder.output_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-18",
      "source": "input",
      "target": "python@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-19",
      "source": "input",
      "target": "word_embeddings",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-20",
      "source": "python@TinyBertPretraining#0",
      "target": "position_embeddings",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-21",
      "source": "word_embeddings",
      "target": "add@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-22",
      "source": "position_embeddings",
      "target": "add@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-23",
      "source": "input",
      "target": "segment_embeddings",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-24",
      "source": "add@TinyBertPretraining#0",
      "target": "add@TinyBertPretraining#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-25",
      "source": "segment_embeddings",
      "target": "add@TinyBertPretraining#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-26",
      "source": "add@TinyBertPretraining#1",
      "target": "embedding_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-27",
      "source": "embedding_norm",
      "target": "encoder.attention.query",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-28",
      "source": "embedding_norm",
      "target": "python@encoder.attention#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-29",
      "source": "embedding_norm",
      "target": "encoder.attention.key",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-30",
      "source": "embedding_norm",
      "target": "python@encoder.attention#1",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-31",
      "source": "embedding_norm",
      "target": "encoder.attention.value",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-32",
      "source": "embedding_norm",
      "target": "python@encoder.attention#2",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-33",
      "source": "embedding_norm",
      "target": "encoder.attention.project",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-34",
      "source": "embedding_norm",
      "target": "add@encoder#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-35",
      "source": "embedding_norm",
      "target": "encoder",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-36",
      "source": "encoder.output_norm",
      "target": "mlm_transform",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-37",
      "source": "encoder",
      "target": "mlm_transform",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-38",
      "source": "mlm_transform",
      "target": "mlm_gelu",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-39",
      "source": "mlm_gelu",
      "target": "mlm_norm",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-40",
      "source": "mlm_norm",
      "target": "mlm_decoder",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-41",
      "source": "encoder.output_norm",
      "target": "pooler",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-42",
      "source": "encoder",
      "target": "pooler",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-43",
      "source": "pooler",
      "target": "pooler_tanh",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-44",
      "source": "pooler_tanh",
      "target": "nsp_head",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-45",
      "source": "mlm_decoder",
      "target": "flatten@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-46",
      "source": "flatten@TinyBertPretraining#0",
      "target": "cat@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-47",
      "source": "nsp_head",
      "target": "cat@TinyBertPretraining#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-48",
      "source": "cat@TinyBertPretraining#0",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "embeddings",
      "target": "word_embeddings",
      "title": "Combine tokens, positions and sentence segments",
      "note": "Each token receives a word embedding, learned absolute-position embedding and A/B segment embedding. Their sum is normalized before entering the encoder. Special IDs mark CLS, SEP and MASK in the recorded examples."
    },
    {
      "id": "bidirectional",
      "target": "encoder.attention.softmax",
      "title": "Consult both earlier and later tokens",
      "note": "This attention is unmasked across all six positions. A later token can influence an earlier representation. Original BERT applies LayerNorm after each attention and feed-forward residual addition."
    },
    {
      "id": "masked",
      "target": "mlm_decoder",
      "title": "Predict only the selected original tokens",
      "note": "The MLM head transforms each position back to vocabulary logits using the same weights as the word embedding. Only selected target positions enter masked-token cross entropy; other predictions are ignored by this loss."
    },
    {
      "id": "pair",
      "target": "nsp_head",
      "title": "Add the original sentence-pair objective",
      "note": "The CLS representation passes through a tanh pooler and a two-class IsNext/NotNext head. The recorded panel exposes MLM, NSP and their summed objective separately. Later BERT-family methods changed this training recipe."
    }
  ],
  "milestone": {
    "name": "BERT pretraining",
    "year": 2018,
    "kind": "Architecture",
    "fidelity": "One post-LN encoder, width eight, two heads, fixed length six and vocabulary sixteen. Token/position/segment embeddings, tied MLM decoder, GELU/LayerNorm MLM transform and a tanh-pooled sentence-pair head.",
    "papers": [
      {
        "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding",
        "url": "https://arxiv.org/abs/1810.04805"
      }
    ]
  },
  "experiment": {
    "kind": "matrices",
    "title": "Use both sides of context, predict selected targets.",
    "description": "Recorded MLM and NSP objectives on synthetic token IDs. The encoder can attend across both segments. Target selection defines the loss; unselected token logits are ignored by MLM. No language understanding is claimed.",
    "controlLabel": "Pretraining example",
    "cases": [
      {
        "id": "mask-a",
        "label": "Mask one token in segment A",
        "target": "mlm_decoder",
        "note": "Toy token IDs: CLS=1, SEP=2 and MASK=3. Segments are A,A,A,A,B,B. Only selected positions contribute to MLM loss. This controlled example always replaces selected words with MASK; original BERT selected 15% of tokens with an 80/10/10 mask/random/unchanged rule. The NSP labels here are illustrative, without a real sentence corpus.",
        "matrices": [
          {
            "label": "Bidirectional attention · head 0",
            "values": [
              [
                0.17624115731229786,
                0.17733530457995425,
                0.17551406588757099,
                0.16281425144362976,
                0.15329032519176883,
                0.15480489558477828
              ],
              [
                0.18778713904040403,
                0.1852502001139755,
                0.17596734922775897,
                0.17206975798323515,
                0.1395970343355672,
                0.1393285192990591
              ],
              [
                0.18642395635738876,
                0.1689461034895644,
                0.17169843251293476,
                0.1782835431439095,
                0.14550324875857662,
                0.14914471573762592
              ],
              [
                0.15077713019335304,
                0.1533873705412099,
                0.14521260863953372,
                0.24600964304811473,
                0.1632639064502691,
                0.14134934112751957
              ],
              [
                0.13881670035267885,
                0.18781574973105694,
                0.1528571591590377,
                0.21762991319505273,
                0.16863852678149718,
                0.13424195078067644
              ],
              [
                0.1861016401413153,
                0.22975338096284326,
                0.19580624258829177,
                0.15892009822985792,
                0.1186173291173773,
                0.11080130896031447
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Original token IDs",
            "values": [
              1,
              4,
              5,
              2,
              6,
              2
            ]
          },
          {
            "label": "Corrupted token IDs",
            "values": [
              1,
              4,
              3,
              2,
              6,
              2
            ]
          },
          {
            "label": "MLM loss positions · 1 means selected",
            "values": [
              0,
              0,
              1,
              0,
              0,
              0
            ]
          },
          {
            "label": "Prediction at first masked position · 16 token probabilities",
            "values": [
              0.0005717056421297077,
              0.7849036643765994,
              0.00014382322697756075,
              0.003132656902926941,
              0.00013850389365066532,
              0.06069144303810712,
              0.0003843040038532849,
              0.00014232709941134925,
              0.00237814889057363,
              0.11143567945665872,
              0.00005625725974555662,
              0.0007285342397772369,
              0.004732973000207605,
              0.002439099928727781,
              0.027591600521638622,
              0.0005292785190145894
            ]
          },
          {
            "label": "Sentence pair · IsNext / NotNext probabilities",
            "values": [
              0.4712098489178174,
              0.5287901510821825
            ]
          }
        ],
        "metrics": [
          {
            "label": "Masked-token cross entropy",
            "value": 2.801952562219697
          },
          {
            "label": "Sentence-pair cross entropy",
            "value": 0.7524517450851487
          },
          {
            "label": "Combined pretraining objective",
            "value": 3.5544043073048455
          }
        ]
      },
      {
        "id": "mask-b",
        "label": "Mask one token in segment B",
        "target": "mlm_decoder",
        "note": "Toy token IDs: CLS=1, SEP=2 and MASK=3. Segments are A,A,A,A,B,B. Only selected positions contribute to MLM loss. This controlled example always replaces selected words with MASK; original BERT selected 15% of tokens with an 80/10/10 mask/random/unchanged rule. The NSP labels here are illustrative, without a real sentence corpus.",
        "matrices": [
          {
            "label": "Bidirectional attention · head 0",
            "values": [
              [
                0.17444152428483226,
                0.17552449900011882,
                0.1768533070224638,
                0.16115172318571075,
                0.1588047940762848,
                0.15322415243058946
              ],
              [
                0.18068737181895325,
                0.178246348278023,
                0.179126153728844,
                0.16556422606142718,
                0.1623150474887786,
                0.13406085262397388
              ],
              [
                0.19447853117655783,
                0.1833016265372508,
                0.18799226825160759,
                0.1549761935191624,
                0.14887090388172045,
                0.13038047663370086
              ],
              [
                0.14022905918428866,
                0.14265669225938066,
                0.13795667341632956,
                0.22879929304040186,
                0.21889746243011235,
                0.131460819669487
              ],
              [
                0.13529006540484417,
                0.15824371044657537,
                0.1490912859217889,
                0.21658940043144836,
                0.21660011954585756,
                0.12418541824948556
              ],
              [
                0.17414090901663293,
                0.21498715744865718,
                0.21146849675688065,
                0.14870632169466952,
                0.14701699357573839,
                0.10368012150742126
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Original token IDs",
            "values": [
              1,
              4,
              5,
              2,
              6,
              2
            ]
          },
          {
            "label": "Corrupted token IDs",
            "values": [
              1,
              4,
              5,
              2,
              3,
              2
            ]
          },
          {
            "label": "MLM loss positions · 1 means selected",
            "values": [
              0,
              0,
              0,
              0,
              1,
              0
            ]
          },
          {
            "label": "Prediction at first masked position · 16 token probabilities",
            "values": [
              0.0021918799848152353,
              0.09592514361677534,
              0.006059176379616244,
              0.0031864662927750913,
              0.0057868174320436245,
              0.1247793551114348,
              0.011106500606571893,
              0.00033778175315408386,
              0.0007530533593815779,
              0.0017176446980809686,
              0.002379938076662047,
              0.00256027120945123,
              0.011458418954539945,
              0.004207832304369409,
              0.7252581556663852,
              0.0022915645539432748
            ]
          },
          {
            "label": "Sentence pair · IsNext / NotNext probabilities",
            "values": [
              0.4749097961954467,
              0.5250902038045532
            ]
          }
        ],
        "metrics": [
          {
            "label": "Masked-token cross entropy",
            "value": 4.5002247018524635
          },
          {
            "label": "Sentence-pair cross entropy",
            "value": 0.7446303957277335
          },
          {
            "label": "Combined pretraining objective",
            "value": 5.244855097580197
          }
        ]
      },
      {
        "id": "both",
        "label": "Mask a token in each segment",
        "target": "mlm_decoder",
        "note": "Toy token IDs: CLS=1, SEP=2 and MASK=3. Segments are A,A,A,A,B,B. Only selected positions contribute to MLM loss. This controlled example always replaces selected words with MASK; original BERT selected 15% of tokens with an 80/10/10 mask/random/unchanged rule. The NSP labels here are illustrative, without a real sentence corpus.",
        "matrices": [
          {
            "label": "Bidirectional attention · head 0",
            "values": [
              [
                0.17498949508482886,
                0.17607587173393566,
                0.17426756745320304,
                0.16165794691332916,
                0.15930364542724634,
                0.1537054733874569
              ],
              [
                0.18247778816723365,
                0.18001257672423684,
                0.17099218211010636,
                0.16720478839872094,
                0.16392341398207735,
                0.13538925061762488
              ],
              [
                0.18203004454112282,
                0.16496413521177625,
                0.1676515933287763,
                0.1740814964640074,
                0.16564327499556272,
                0.14562945545875455
              ],
              [
                0.1406373085920805,
                0.14307200924482733,
                0.13544700331217363,
                0.22946539731600843,
                0.2195347394675828,
                0.13184354206732732
              ],
              [
                0.1368109670660509,
                0.16002265202202304,
                0.139525554152023,
                0.21902425163748643,
                0.219035091253862,
                0.1255814838685545
              ],
              [
                0.17920279646520407,
                0.2212363541482644,
                0.1885476463857135,
                0.1530288824735244,
                0.15129044935766847,
                0.10669387116962509
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Original token IDs",
            "values": [
              1,
              4,
              5,
              2,
              6,
              2
            ]
          },
          {
            "label": "Corrupted token IDs",
            "values": [
              1,
              4,
              3,
              2,
              3,
              2
            ]
          },
          {
            "label": "MLM loss positions · 1 means selected",
            "values": [
              0,
              0,
              1,
              0,
              1,
              0
            ]
          },
          {
            "label": "Prediction at first masked position · 16 token probabilities",
            "values": [
              0.0005277966340641987,
              0.7852787306526695,
              0.0001228819079733716,
              0.003022497760007055,
              0.000143051707015915,
              0.05573067746615639,
              0.00034727954014823244,
              0.0001279839508834509,
              0.0020021580748270837,
              0.12035672165153606,
              0.00005149574613845328,
              0.0007017570663769098,
              0.003828507292841028,
              0.0023719743790728345,
              0.024896108086887404,
              0.0004903780834018589
            ]
          },
          {
            "label": "Sentence pair · IsNext / NotNext probabilities",
            "values": [
              0.4734943958901561,
              0.5265056041098438
            ]
          }
        ],
        "metrics": [
          {
            "label": "Masked-token cross entropy",
            "value": 3.7182218840498935
          },
          {
            "label": "Sentence-pair cross entropy",
            "value": 0.7476152019272986
          },
          {
            "label": "Combined pretraining objective",
            "value": 4.465837085977192
          }
        ]
      },
      {
        "id": "not-next",
        "label": "Same tokens with a negative sentence-pair label",
        "target": "mlm_decoder",
        "note": "Toy token IDs: CLS=1, SEP=2 and MASK=3. Segments are A,A,A,A,B,B. Only selected positions contribute to MLM loss. This controlled example always replaces selected words with MASK; original BERT selected 15% of tokens with an 80/10/10 mask/random/unchanged rule. The NSP labels here are illustrative, without a real sentence corpus.",
        "matrices": [
          {
            "label": "Bidirectional attention · head 0",
            "values": [
              [
                0.17624115731229786,
                0.17733530457995425,
                0.17551406588757099,
                0.16281425144362976,
                0.15329032519176883,
                0.15480489558477828
              ],
              [
                0.18778713904040403,
                0.1852502001139755,
                0.17596734922775897,
                0.17206975798323515,
                0.1395970343355672,
                0.1393285192990591
              ],
              [
                0.18642395635738876,
                0.1689461034895644,
                0.17169843251293476,
                0.1782835431439095,
                0.14550324875857662,
                0.14914471573762592
              ],
              [
                0.15077713019335304,
                0.1533873705412099,
                0.14521260863953372,
                0.24600964304811473,
                0.1632639064502691,
                0.14134934112751957
              ],
              [
                0.13881670035267885,
                0.18781574973105694,
                0.1528571591590377,
                0.21762991319505273,
                0.16863852678149718,
                0.13424195078067644
              ],
              [
                0.1861016401413153,
                0.22975338096284326,
                0.19580624258829177,
                0.15892009822985792,
                0.1186173291173773,
                0.11080130896031447
              ]
            ]
          }
        ],
        "vectors": [
          {
            "label": "Original token IDs",
            "values": [
              1,
              4,
              5,
              2,
              6,
              2
            ]
          },
          {
            "label": "Corrupted token IDs",
            "values": [
              1,
              4,
              3,
              2,
              6,
              2
            ]
          },
          {
            "label": "MLM loss positions · 1 means selected",
            "values": [
              0,
              0,
              1,
              0,
              0,
              0
            ]
          },
          {
            "label": "Prediction at first masked position · 16 token probabilities",
            "values": [
              0.0005717056421297077,
              0.7849036643765994,
              0.00014382322697756075,
              0.003132656902926941,
              0.00013850389365066532,
              0.06069144303810712,
              0.0003843040038532849,
              0.00014232709941134925,
              0.00237814889057363,
              0.11143567945665872,
              0.00005625725974555662,
              0.0007285342397772369,
              0.004732973000207605,
              0.002439099928727781,
              0.027591600521638622,
              0.0005292785190145894
            ]
          },
          {
            "label": "Sentence pair · IsNext / NotNext probabilities",
            "values": [
              0.4712098489178174,
              0.5287901510821825
            ]
          }
        ],
        "metrics": [
          {
            "label": "Masked-token cross entropy",
            "value": 2.801952562219697
          },
          {
            "label": "Sentence-pair cross entropy",
            "value": 0.6371636156518028
          },
          {
            "label": "Combined pretraining objective",
            "value": 3.4391161778715
          }
        ]
      }
    ]
  },
  "revision": "0c48027dcf5a807d",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/bert/0c48027dcf5a807d/graph.tensorviz.json",
      "sha256": "65c731b79ab674c1effceb7f16230cd4e2b85161df198f99695294c9872c3644",
      "bytes": 87860
    },
    "model.py": {
      "url": "/models/bert/0c48027dcf5a807d/model.py",
      "sha256": "fa041d05ccb2a15cb1e139d53180ab5ca4930dd11ba12f3fd3aec170228f0fc5",
      "bytes": 8698
    },
    "run.py": {
      "url": "/models/bert/0c48027dcf5a807d/run.py",
      "sha256": "f066cf76617849deec718f7ec547ef7fc024018bca9b9e6d3eba407daea1d238",
      "bytes": 680
    },
    "requirements.txt": {
      "url": "/models/bert/0c48027dcf5a807d/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/bert/0c48027dcf5a807d/preview.svg",
      "sha256": "e17f878735ca3b5be513e6f9fe279b75df8520ec51d4d37ccc4a178b7190afed",
      "bytes": 1750
    }
  }
}