{
  "schemaVersion": 1,
  "slug": "lora",
  "title": "Learn a small update to frozen weights",
  "className": "LowRankAdapter",
  "question": "How can two small matrices adapt a frozen linear layer?",
  "description": "Follow the frozen base and rank-two adapter, inspect actual toy training checkpoints, and verify the merged output.",
  "category": "Modern LLMs",
  "tags": [
    "LoRA",
    "Low rank",
    "Fine-tuning",
    "Frozen weights"
  ],
  "curator": "TensorViz",
  "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.",
  "license": "No separate redistribution license has been declared for these project examples.",
  "limitations": "The base is randomly initialized, not pretrained. Toy fitting demonstrates low-rank adaptation and gradients; it does not reproduce language-model fine-tuning. Benefits depend on chosen rank and adapted layers.",
  "openingStep": "rank",
  "provenance": {
    "capturedAt": "2026-09-17T06:03:06.039514+00:00",
    "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c",
    "sourceSha256": "2348feed277b52dd4ab33eecebfd6fbf9a677b37afd2da5376c09ae104dcdf88",
    "python": "3.13.13",
    "torch": "2.7.1",
    "execution": {
      "inputs": {
        "args": [
          {
            "shape": [
              1,
              4,
              8
            ],
            "dtype": "float32",
            "fill": "randn"
          }
        ]
      },
      "device": "cpu",
      "mode": "eval",
      "seed": 0,
      "deterministic": true
    },
    "outcome": "passed",
    "outputShape": [
      1,
      4,
      8
    ],
    "method": "TensorViz multifile projection and isolated execution worker",
    "scope": "Original width-8 rank-2 LoRA layer. Exact merge and gradient checks plus deterministic SGD on an eight-example synthetic task.",
    "numericalChecks": [
      "Zero-B initialization exactly preserves the frozen base output",
      "Only B has a nonzero adapter gradient at the first step; the base receives no gradient",
      "Toy training reduces fitting loss while base weights remain byte-identical",
      "Merged weights reproduce the two-branch output and the update rank is at most two"
    ]
  },
  "layers": [
    {
      "id": "input",
      "label": "input",
      "op": "Input",
      "kind": "input",
      "parent": null,
      "params": {
        "shape": "1,4,8"
      }
    },
    {
      "id": "base",
      "label": "base",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 14,
        "endLine": 14
      }
    },
    {
      "id": "down",
      "label": "down",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 8,
        "out_features": 2,
        "bias": false
      },
      "source": {
        "line": 16,
        "endLine": 16
      }
    },
    {
      "id": "up",
      "label": "up",
      "op": "Linear",
      "kind": "layer",
      "parent": null,
      "params": {
        "in_features": 2,
        "out_features": 8,
        "bias": false
      },
      "source": {
        "line": 17,
        "endLine": 17
      }
    },
    {
      "id": "add@LowRankAdapter#0",
      "label": "add@LowRankAdapter#0",
      "op": "torch.add",
      "kind": "layer",
      "parent": null,
      "params": {},
      "source": {
        "line": 24,
        "endLine": 24
      }
    },
    {
      "id": "output",
      "label": "output",
      "op": "Output",
      "kind": "output",
      "parent": null,
      "params": {}
    }
  ],
  "connections": [
    {
      "id": "edge-1",
      "source": "input",
      "target": "base",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-2",
      "source": "input",
      "target": "down",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-3",
      "source": "down",
      "target": "up",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-4",
      "source": "base",
      "target": "add@LowRankAdapter#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-5",
      "source": "up",
      "target": "add@LowRankAdapter#0",
      "sourcePort": null,
      "targetPort": null
    },
    {
      "id": "edge-6",
      "source": "add@LowRankAdapter#0",
      "target": "output",
      "sourcePort": null,
      "targetPort": null
    }
  ],
  "steps": [
    {
      "id": "frozen",
      "target": "base",
      "title": "Keep the base weights frozen",
      "note": "The 8 × 8 base has 64 parameters with requires_grad=False. Its output is still part of the forward calculation, but the optimizer never updates its weights."
    },
    {
      "id": "rank",
      "target": "down",
      "title": "Project into a rank-two subspace",
      "note": "The trainable A matrix maps eight features to two. This limits the effective update BA to rank at most two. Rank bounds the weight update, not the rank of the complete base matrix."
    },
    {
      "id": "up",
      "target": "up",
      "title": "Return to eight features",
      "note": "B maps two features back to eight and starts at zero. The adapter initially changes nothing. B receives the first update; A's first gradient is zero until B becomes nonzero."
    },
    {
      "id": "merge",
      "target": "add@LowRankAdapter#0",
      "title": "Add the update, or merge its weights",
      "note": "The two branches add as Wx + (alpha/r)BAx. At inference the weights can be merged as W + (alpha/r)BA. The recorded checkpoints check the resulting outputs match."
    }
  ],
  "milestone": {
    "name": "LoRA",
    "year": 2021,
    "kind": "Training method",
    "fidelity": "One frozen bias-free 8 × 8 base matrix, trainable A (2 × 8) and B (8 × 2), alpha = rank = 2. Standard random-A / zero-B initialization.",
    "papers": [
      {
        "title": "LoRA: Low-Rank Adaptation of Large Language Models",
        "url": "https://arxiv.org/abs/2106.09685"
      }
    ]
  },
  "experiment": {
    "kind": "vectors",
    "title": "Fit an update, then merge the branches.",
    "description": "Recorded deterministic SGD on a synthetic rank-two task. W + (alpha/r) B A gives the same output after merging, up to floating-point rounding.",
    "controlLabel": "Recorded training checkpoint",
    "cases": [
      {
        "id": "step-0",
        "label": "After 0 SGD steps",
        "target": "up",
        "note": "A fixed eight-example toy task asks for a rank-two weight change. The base is frozen throughout; only A (8 → 2) and B (2 → 8) are fitted. At initialization B is zero. This records adapter arithmetic, not fine-tuning quality on language tasks.",
        "vectors": [
          {
            "label": "Frozen base · first example",
            "values": [
              0.23352333903312683,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Adapter update",
            "values": [
              0,
              0,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Adapted output",
            "values": [
              0.23352333903312683,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Toy target",
            "values": [
              1.0335233390331269,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          }
        ],
        "metrics": [
          {
            "label": "Mean squared error · all eight examples",
            "value": 0.015625
          },
          {
            "label": "Maximum merge error",
            "value": 0
          },
          {
            "label": "Trainable adapter parameters",
            "value": 32
          },
          {
            "label": "Frozen base parameters",
            "value": 64
          }
        ]
      },
      {
        "id": "step-1",
        "label": "After 1 SGD steps",
        "target": "up",
        "note": "A fixed eight-example toy task asks for a rank-two weight change. The base is frozen throughout; only A (8 → 2) and B (2 → 8) are fitted. At initialization B is zero. This records adapter arithmetic, not fine-tuning quality on language tasks.",
        "vectors": [
          {
            "label": "Frozen base · first example",
            "values": [
              0.23352333903312683,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Adapter update",
            "values": [
              0.001524511580712068,
              0.0015704850457646247,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Adapted output",
            "values": [
              0.2350478506138389,
              -0.32846123773827773,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Toy target",
            "values": [
              1.0335233390331269,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          }
        ],
        "metrics": [
          {
            "label": "Mean squared error · all eight examples",
            "value": 0.015533716280293791
          },
          {
            "label": "Maximum merge error",
            "value": 0
          },
          {
            "label": "Trainable adapter parameters",
            "value": 32
          },
          {
            "label": "Frozen base parameters",
            "value": 64
          }
        ]
      },
      {
        "id": "step-40",
        "label": "After 40 SGD steps",
        "target": "up",
        "note": "A fixed eight-example toy task asks for a rank-two weight change. The base is frozen throughout; only A (8 → 2) and B (2 → 8) are fitted. At initialization B is zero. This records adapter arithmetic, not fine-tuning quality on language tasks.",
        "vectors": [
          {
            "label": "Frozen base · first example",
            "values": [
              0.23352333903312683,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Adapter update",
            "values": [
              0.07390527149455983,
              0.06768452751183007,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Adapted output",
            "values": [
              0.30742861052768666,
              -0.2623471952722123,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Toy target",
            "values": [
              1.0335233390331269,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          }
        ],
        "metrics": [
          {
            "label": "Mean squared error · all eight examples",
            "value": 0.01263677016308464
          },
          {
            "label": "Maximum merge error",
            "value": 0
          },
          {
            "label": "Trainable adapter parameters",
            "value": 32
          },
          {
            "label": "Frozen base parameters",
            "value": 64
          }
        ]
      },
      {
        "id": "step-320",
        "label": "After 320 SGD steps",
        "target": "up",
        "note": "A fixed eight-example toy task asks for a rank-two weight change. The base is frozen throughout; only A (8 → 2) and B (2 → 8) are fitted. At initialization B is zero. This records adapter arithmetic, not fine-tuning quality on language tasks.",
        "vectors": [
          {
            "label": "Frozen base · first example",
            "values": [
              0.23352333903312683,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Adapter update",
            "values": [
              0.7998001793473433,
              0.0010423141350499043,
              0,
              0,
              0,
              0,
              0,
              0
            ]
          },
          {
            "label": "Adapted output",
            "values": [
              1.03332351838047,
              -0.32898940864899245,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          },
          {
            "label": "Toy target",
            "values": [
              1.0335233390331269,
              -0.33003172278404236,
              0.34006670117378235,
              0.335989773273468,
              -0.3223230242729187,
              0.05789417400956154,
              0.25199094414711,
              0.22488880157470703
            ]
          }
        ],
        "metrics": [
          {
            "label": "Mean squared error · all eight examples",
            "value": 0.00002589255714517887
          },
          {
            "label": "Maximum merge error",
            "value": 0
          },
          {
            "label": "Trainable adapter parameters",
            "value": 32
          },
          {
            "label": "Frozen base parameters",
            "value": 64
          }
        ]
      }
    ]
  },
  "revision": "a844ca6fe86121f9",
  "files": {
    "graph.tensorviz.json": {
      "url": "/models/lora/a844ca6fe86121f9/graph.tensorviz.json",
      "sha256": "42e9de2df3c8d034fd9ac40c78d4d92c0a50614dc7abd0ac58177405a8b74b81",
      "bytes": 20338
    },
    "model.py": {
      "url": "/models/lora/a844ca6fe86121f9/model.py",
      "sha256": "2348feed277b52dd4ab33eecebfd6fbf9a677b37afd2da5376c09ae104dcdf88",
      "bytes": 4103
    },
    "run.py": {
      "url": "/models/lora/a844ca6fe86121f9/run.py",
      "sha256": "e2a1ff8dabc1b2e52f3c27ea0a039b21ae5c87eab95686f33f3de8a94ce87f13",
      "bytes": 652
    },
    "requirements.txt": {
      "url": "/models/lora/a844ca6fe86121f9/requirements.txt",
      "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf",
      "bytes": 91
    },
    "preview.svg": {
      "url": "/models/lora/a844ca6fe86121f9/preview.svg",
      "sha256": "7673f0587390f3d87a66d044c3aa59dd782ce52603e12598e62e2467540c55f9",
      "bytes": 2308
    }
  }
}