{ "schemaVersion": 1, "slug": "clip", "title": "Meet in a shared image–text space", "className": "TinyCLIP", "question": "Where do the image and text meet?", "description": "Trace two independent encoders into normalized embeddings, then compare both directions of the image–text contrastive objective.", "category": "Multimodal milestones", "tags": [ "CLIP", "Contrastive learning", "Dual encoder", "Temperature" ], "curator": "TensorViz", "attribution": "Original TensorViz teaching example. PyTorch provides the underlying operators.", "license": "No separate redistribution license has been declared for these project examples.", "limitations": "Reduced untrained ViT/text towers with a vocabulary of sixteen, fixed-length inputs, GELU and no tokenizer or augmentation pipeline. Text pooling assumes EOS occupies the final position. Synthetic pairs teach the objective, not image–text understanding or zero-shot accuracy.", "openingStep": "compare", "provenance": { "capturedAt": "2026-09-17T06:47:49.423016+00:00", "productRevision": "cf0ab1c9c58c1dc640235b742fa032cade00b43c", "sourceSha256": "a7e72c3bbad2e899038b77cab7a58e4115cbe191500180e55a0058316f782711", "python": "3.13.13", "torch": "2.7.1", "execution": { "inputs": { "args": [ { "shape": [ 3, 3, 8, 8 ], "dtype": "float32", "fill": "randn" }, { "shape": [ 3, 4 ], "dtype": "int64", "fill": "ones" } ] }, "device": "cpu", "mode": "eval", "seed": 0, "deterministic": true }, "outcome": "passed", "outputShape": [ 3, 3 ], "method": "TensorViz multifile projection and isolated execution worker", "scope": "Reduced image/text encoders, normalized cosine scores, symmetric contrastive loss, pairing permutations, causal text visibility and gradient checks. Synthetic images and token IDs; no trained semantic alignment.", "numericalChecks": [ "Both projected modalities have unit norm; logits are scaled pairwise cosines", "Joint pair permutations preserve the objective; text-only permutations move similarity columns", "The text attention is causal and the symmetric loss equals both diagonal log-probabilities", "Both encoders and the learned log-temperature receive gradients; a small SGD step lowers this batch loss" ] }, "layers": [ { "id": "input", "label": "input", "op": "Input", "kind": "input", "parent": null, "params": { "shape": "3,3,8,8" } }, { "id": "image_encoder", "label": "image_encoder", "op": "Module", "kind": "module", "parent": null, "params": {}, "source": { "line": 88, "endLine": 88 } }, { "id": "text_encoder", "label": "text_encoder", "op": "Module", "kind": "module", "parent": null, "params": {}, "source": { "line": 89, "endLine": 89 } }, { "id": "similarity", "label": "similarity", "op": "Module", "kind": "module", "parent": null, "params": {}, "source": { "line": 90, "endLine": 90 } }, { "id": "output", "label": "output", "op": "Output", "kind": "output", "parent": null, "params": {} }, { "id": "image_encoder.patches", "label": "image_encoder.patches", "op": "Conv2d", "kind": "layer", "parent": "image_encoder", "params": { "in_channels": 3, "out_channels": 8, "kernel_size": 4, "stride": 4, "bias": false }, "source": { "line": 47, "endLine": 47 } }, { "id": "flatten@image_encoder#0", "label": "flatten@image_encoder#0", "op": "Flatten", "kind": "layer", "parent": "image_encoder", "params": { "start_dim": 2 }, "source": { "line": 56, "endLine": 56 } }, { "id": "python@image_encoder#0", "label": "python@image_encoder#0", "op": "Python", "kind": "layer", "parent": "image_encoder", "params": {}, "source": { "line": 56, "endLine": 56 } }, { "id": "image_encoder.class_token", "label": "image_encoder.class_token", "op": "Parameter", "kind": "layer", "parent": "image_encoder", "params": {}, "source": { "line": 48, "endLine": 48 } }, { "id": "cat@image_encoder#0", "label": "cat@image_encoder#0", "op": "torch.cat", "kind": "layer", "parent": "image_encoder", "params": { "dim": 1 }, "source": { "line": 57, "endLine": 57 } }, { "id": "image_encoder.positions", "label": "image_encoder.positions", "op": "Parameter", "kind": "layer", "parent": "image_encoder", "params": {}, "source": { "line": 49, "endLine": 49 } }, { "id": "add@image_encoder#0", "label": "add@image_encoder#0", "op": "torch.add", "kind": "layer", "parent": "image_encoder", "params": {}, "source": { "line": 58, "endLine": 58 } }, { "id": "image_encoder.pre_norm", "label": "image_encoder.pre_norm", "op": "LayerNorm", "kind": "layer", "parent": "image_encoder", "params": { "normalized_shape": 8 }, "source": { "line": 50, "endLine": 50 } }, { "id": "image_encoder.block", "label": "image_encoder.block", "op": "Module", "kind": "module", "parent": "image_encoder", "params": {}, "source": { "line": 51, "endLine": 51 } }, { "id": "image_encoder.post_norm", "label": "image_encoder.post_norm", "op": "LayerNorm", "kind": "layer", "parent": "image_encoder", "params": { "normalized_shape": 8 }, "source": { "line": 52, "endLine": 52 } }, { "id": "image_encoder.project", "label": "image_encoder.project", "op": "Linear", "kind": "layer", "parent": "image_encoder", "params": { "in_features": 8, "out_features": 4, "bias": false }, "source": { "line": 53, "endLine": 53 } }, { "id": "python@image_encoder#1", "label": "python@image_encoder#1", "op": "Python", "kind": "layer", "parent": "image_encoder", "params": {}, "source": { "line": 59, "endLine": 59 } }, { "id": "image_encoder.block.norm", "label": "image_encoder.block.norm", "op": "LayerNorm", "kind": "layer", "parent": "image_encoder.block", "params": { "normalized_shape": 8 }, "source": { "line": 17, "endLine": 17 } }, { "id": "image_encoder.block.query", "label": "image_encoder.block.query", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 18, "endLine": 18 } }, { "id": "python@image_encoder.block#0", "label": "python@image_encoder.block#0", "op": "Python", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "image_encoder.block.key", "label": "image_encoder.block.key", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 19, "endLine": 19 } }, { "id": "python@image_encoder.block#1", "label": "python@image_encoder.block#1", "op": "Python", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 32, "endLine": 32 } }, { "id": "div@image_encoder.block#0", "label": "div@image_encoder.block#0", "op": "torch.div", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "python@image_encoder.block#3", "label": "python@image_encoder.block#3", "op": "Python", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 36, "endLine": 38 } }, { "id": "image_encoder.block.value", "label": "image_encoder.block.value", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 20, "endLine": 20 } }, { "id": "python@image_encoder.block#2", "label": "python@image_encoder.block#2", "op": "Python", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "image_encoder.block.softmax", "label": "image_encoder.block.softmax", "op": "Softmax", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "python@image_encoder.block#4", "label": "python@image_encoder.block#4", "op": "Python", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 39, "endLine": 39 } }, { "id": "image_encoder.block.project", "label": "image_encoder.block.project", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 22, "endLine": 22 } }, { "id": "add@image_encoder.block#0", "label": "add@image_encoder.block#0", "op": "torch.add", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 40, "endLine": 40 } }, { "id": "image_encoder.block.ff_norm", "label": "image_encoder.block.ff_norm", "op": "LayerNorm", "kind": "layer", "parent": "image_encoder.block", "params": { "normalized_shape": 8 }, "source": { "line": 23, "endLine": 23 } }, { "id": "image_encoder.block.up", "label": "image_encoder.block.up", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 8, "out_features": 32 }, "source": { "line": 24, "endLine": 24 } }, { "id": "image_encoder.block.gelu", "label": "image_encoder.block.gelu", "op": "GELU", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 25, "endLine": 25 } }, { "id": "image_encoder.block.down", "label": "image_encoder.block.down", "op": "Linear", "kind": "layer", "parent": "image_encoder.block", "params": { "in_features": 32, "out_features": 8 }, "source": { "line": 26, "endLine": 26 } }, { "id": "add@image_encoder.block#1", "label": "add@image_encoder.block#1", "op": "torch.add", "kind": "layer", "parent": "image_encoder.block", "params": {}, "source": { "line": 41, "endLine": 41 } }, { "id": "text_encoder.tokens", "label": "text_encoder.tokens", "op": "Embedding", "kind": "layer", "parent": "text_encoder", "params": { "num_embeddings": 16, "embedding_dim": 8 }, "source": { "line": 65, "endLine": 65 } }, { "id": "text_encoder.positions", "label": "text_encoder.positions", "op": "Parameter", "kind": "layer", "parent": "text_encoder", "params": {}, "source": { "line": 66, "endLine": 66 } }, { "id": "add@text_encoder#0", "label": "add@text_encoder#0", "op": "torch.add", "kind": "layer", "parent": "text_encoder", "params": {}, "source": { "line": 72, "endLine": 72 } }, { "id": "text_encoder.block", "label": "text_encoder.block", "op": "Module", "kind": "module", "parent": "text_encoder", "params": { "causal": true }, "source": { "line": 67, "endLine": 67 } }, { "id": "text_encoder.norm", "label": "text_encoder.norm", "op": "LayerNorm", "kind": "layer", "parent": "text_encoder", "params": { "normalized_shape": 8 }, "source": { "line": 68, "endLine": 68 } }, { "id": "text_encoder.project", "label": "text_encoder.project", "op": "Linear", "kind": "layer", "parent": "text_encoder", "params": { "in_features": 8, "out_features": 4, "bias": false }, "source": { "line": 69, "endLine": 69 } }, { "id": "python@text_encoder#0", "label": "python@text_encoder#0", "op": "Python", "kind": "layer", "parent": "text_encoder", "params": {}, "source": { "line": 73, "endLine": 73 } }, { "id": "text_encoder.block.norm", "label": "text_encoder.block.norm", "op": "LayerNorm", "kind": "layer", "parent": "text_encoder.block", "params": { "normalized_shape": 8 }, "source": { "line": 17, "endLine": 17 } }, { "id": "text_encoder.block.query", "label": "text_encoder.block.query", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 18, "endLine": 18 } }, { "id": "python@text_encoder.block#0", "label": "python@text_encoder.block#0", "op": "Python", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 31, "endLine": 31 } }, { "id": "text_encoder.block.key", "label": "text_encoder.block.key", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 19, "endLine": 19 } }, { "id": "python@text_encoder.block#1", "label": "python@text_encoder.block#1", "op": "Python", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 32, "endLine": 32 } }, { "id": "div@text_encoder.block#0", "label": "div@text_encoder.block#0", "op": "torch.div", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 34, "endLine": 34 } }, { "id": "python@text_encoder.block#3", "label": "python@text_encoder.block#3", "op": "Python", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 36, "endLine": 38 } }, { "id": "text_encoder.block.value", "label": "text_encoder.block.value", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 20, "endLine": 20 } }, { "id": "python@text_encoder.block#2", "label": "python@text_encoder.block#2", "op": "Python", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 33, "endLine": 33 } }, { "id": "text_encoder.block.softmax", "label": "text_encoder.block.softmax", "op": "Softmax", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 21, "endLine": 21 } }, { "id": "python@text_encoder.block#4", "label": "python@text_encoder.block#4", "op": "Python", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 39, "endLine": 39 } }, { "id": "text_encoder.block.project", "label": "text_encoder.block.project", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 8, "out_features": 8 }, "source": { "line": 22, "endLine": 22 } }, { "id": "add@text_encoder.block#0", "label": "add@text_encoder.block#0", "op": "torch.add", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 40, "endLine": 40 } }, { "id": "text_encoder.block.ff_norm", "label": "text_encoder.block.ff_norm", "op": "LayerNorm", "kind": "layer", "parent": "text_encoder.block", "params": { "normalized_shape": 8 }, "source": { "line": 23, "endLine": 23 } }, { "id": "text_encoder.block.up", "label": "text_encoder.block.up", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 8, "out_features": 32 }, "source": { "line": 24, "endLine": 24 } }, { "id": "text_encoder.block.gelu", "label": "text_encoder.block.gelu", "op": "GELU", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 25, "endLine": 25 } }, { "id": "text_encoder.block.down", "label": "text_encoder.block.down", "op": "Linear", "kind": "layer", "parent": "text_encoder.block", "params": { "in_features": 32, "out_features": 8 }, "source": { "line": 26, "endLine": 26 } }, { "id": "add@text_encoder.block#1", "label": "add@text_encoder.block#1", "op": "torch.add", "kind": "layer", "parent": "text_encoder.block", "params": {}, "source": { "line": 41, "endLine": 41 } }, { "id": "similarity.log_scale", "label": "similarity.log_scale", "op": "Parameter", "kind": "layer", "parent": "similarity", "params": {}, "source": { "line": 79, "endLine": 79 } }, { "id": "mul@similarity#0", "label": "mul@similarity#0", "op": "torch.mul", "kind": "layer", "parent": "similarity", "params": {}, "source": { "line": 82, "endLine": 82 } } ], "connections": [ { "id": "edge-1", "source": "image_encoder.block.norm", "target": "image_encoder.block.query", "sourcePort": null, "targetPort": null }, { "id": "edge-2", "source": "image_encoder.block.query", "target": "python@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-3", "source": "image_encoder.block.norm", "target": "image_encoder.block.key", "sourcePort": null, "targetPort": null }, { "id": "edge-4", "source": "image_encoder.block.key", "target": "python@image_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-5", "source": "image_encoder.block.norm", "target": "image_encoder.block.value", "sourcePort": null, "targetPort": null }, { "id": "edge-6", "source": "image_encoder.block.value", "target": "python@image_encoder.block#2", "sourcePort": null, "targetPort": null }, { "id": "edge-7", "source": "python@image_encoder.block#0", "target": "div@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-8", "source": "python@image_encoder.block#1", "target": "div@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-9", "source": "div@image_encoder.block#0", "target": "python@image_encoder.block#3", "sourcePort": null, "targetPort": null }, { "id": "edge-10", "source": "python@image_encoder.block#3", "target": "image_encoder.block.softmax", "sourcePort": null, "targetPort": null }, { "id": "edge-11", "source": "image_encoder.block.softmax", "target": "python@image_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-12", "source": "python@image_encoder.block#2", "target": "python@image_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-13", "source": "python@image_encoder.block#4", "target": "image_encoder.block.project", "sourcePort": null, "targetPort": null }, { "id": "edge-14", "source": "image_encoder.block.project", "target": "add@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-15", "source": "add@image_encoder.block#0", "target": "image_encoder.block.ff_norm", "sourcePort": null, "targetPort": null }, { "id": "edge-16", "source": "image_encoder.block.ff_norm", "target": "image_encoder.block.up", "sourcePort": null, "targetPort": null }, { "id": "edge-17", "source": "image_encoder.block.up", "target": "image_encoder.block.gelu", "sourcePort": null, "targetPort": null }, { "id": "edge-18", "source": "image_encoder.block.gelu", "target": "image_encoder.block.down", "sourcePort": null, "targetPort": null }, { "id": "edge-19", "source": "add@image_encoder.block#0", "target": "add@image_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-20", "source": "image_encoder.block.down", "target": "add@image_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-21", "source": "image_encoder.patches", "target": "flatten@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-22", "source": "flatten@image_encoder#0", "target": "python@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-23", "source": "image_encoder.class_token", "target": "cat@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-24", "source": "python@image_encoder#0", "target": "cat@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-25", "source": "cat@image_encoder#0", "target": "add@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-26", "source": "image_encoder.positions", "target": "add@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-27", "source": "add@image_encoder#0", "target": "image_encoder.pre_norm", "sourcePort": null, "targetPort": null }, { "id": "edge-28", "source": "image_encoder.pre_norm", "target": "image_encoder.block.norm", "sourcePort": null, "targetPort": null }, { "id": "edge-29", "source": "image_encoder.pre_norm", "target": "python@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-30", "source": "image_encoder.pre_norm", "target": "python@image_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-31", "source": "image_encoder.pre_norm", "target": "python@image_encoder.block#2", "sourcePort": null, "targetPort": null }, { "id": "edge-32", "source": "image_encoder.pre_norm", "target": "python@image_encoder.block#3", "sourcePort": null, "targetPort": null }, { "id": "edge-33", "source": "image_encoder.pre_norm", "target": "python@image_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-34", "source": "image_encoder.pre_norm", "target": "add@image_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-35", "source": "image_encoder.pre_norm", "target": "image_encoder.block", "sourcePort": null, "targetPort": null }, { "id": "edge-36", "source": "add@image_encoder.block#1", "target": "image_encoder.post_norm", "sourcePort": null, "targetPort": null }, { "id": "edge-37", "source": "image_encoder.block", "target": "image_encoder.post_norm", "sourcePort": null, "targetPort": null }, { "id": "edge-38", "source": "image_encoder.post_norm", "target": "image_encoder.project", "sourcePort": null, "targetPort": null }, { "id": "edge-39", "source": "image_encoder.project", "target": "python@image_encoder#1", "sourcePort": null, "targetPort": null }, { "id": "edge-40", "source": "text_encoder.block.norm", "target": "text_encoder.block.query", "sourcePort": null, "targetPort": null }, { "id": "edge-41", "source": "text_encoder.block.query", "target": "python@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-42", "source": "text_encoder.block.norm", "target": "text_encoder.block.key", "sourcePort": null, "targetPort": null }, { "id": "edge-43", "source": "text_encoder.block.key", "target": "python@text_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-44", "source": "text_encoder.block.norm", "target": "text_encoder.block.value", "sourcePort": null, "targetPort": null }, { "id": "edge-45", "source": "text_encoder.block.value", "target": "python@text_encoder.block#2", "sourcePort": null, "targetPort": null }, { "id": "edge-46", "source": "python@text_encoder.block#0", "target": "div@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-47", "source": "python@text_encoder.block#1", "target": "div@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-48", "source": "div@text_encoder.block#0", "target": "python@text_encoder.block#3", "sourcePort": null, "targetPort": null }, { "id": "edge-49", "source": "python@text_encoder.block#3", "target": "text_encoder.block.softmax", "sourcePort": null, "targetPort": null }, { "id": "edge-50", "source": "text_encoder.block.softmax", "target": "python@text_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-51", "source": "python@text_encoder.block#2", "target": "python@text_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-52", "source": "python@text_encoder.block#4", "target": "text_encoder.block.project", "sourcePort": null, "targetPort": null }, { "id": "edge-53", "source": "text_encoder.block.project", "target": "add@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-54", "source": "add@text_encoder.block#0", "target": "text_encoder.block.ff_norm", "sourcePort": null, "targetPort": null }, { "id": "edge-55", "source": "text_encoder.block.ff_norm", "target": "text_encoder.block.up", "sourcePort": null, "targetPort": null }, { "id": "edge-56", "source": "text_encoder.block.up", "target": "text_encoder.block.gelu", "sourcePort": null, "targetPort": null }, { "id": "edge-57", "source": "text_encoder.block.gelu", "target": "text_encoder.block.down", "sourcePort": null, "targetPort": null }, { "id": "edge-58", "source": "add@text_encoder.block#0", "target": "add@text_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-59", "source": "text_encoder.block.down", "target": "add@text_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-60", "source": "text_encoder.tokens", "target": "add@text_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-61", "source": "text_encoder.positions", "target": "add@text_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-62", "source": "add@text_encoder#0", "target": "text_encoder.block.norm", "sourcePort": null, "targetPort": null }, { "id": "edge-63", "source": "add@text_encoder#0", "target": "python@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-64", "source": "add@text_encoder#0", "target": "python@text_encoder.block#1", "sourcePort": null, "targetPort": null }, { "id": "edge-65", "source": "add@text_encoder#0", "target": "python@text_encoder.block#2", "sourcePort": null, "targetPort": null }, { "id": "edge-66", "source": "add@text_encoder#0", "target": "python@text_encoder.block#3", "sourcePort": null, "targetPort": null }, { "id": "edge-67", "source": "add@text_encoder#0", "target": "python@text_encoder.block#4", "sourcePort": null, "targetPort": null }, { "id": "edge-68", "source": "add@text_encoder#0", "target": "add@text_encoder.block#0", "sourcePort": null, "targetPort": null }, { "id": "edge-69", "source": "add@text_encoder#0", "target": "text_encoder.block", "sourcePort": null, "targetPort": null }, { "id": "edge-70", "source": "add@text_encoder.block#1", "target": "text_encoder.norm", "sourcePort": null, "targetPort": null }, { "id": "edge-71", "source": "text_encoder.block", "target": "text_encoder.norm", "sourcePort": null, "targetPort": null }, { "id": "edge-72", "source": "text_encoder.norm", "target": "text_encoder.project", "sourcePort": null, "targetPort": null }, { "id": "edge-73", "source": "text_encoder.project", "target": "python@text_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-74", "source": "similarity.log_scale", "target": "mul@similarity#0", "sourcePort": null, "targetPort": null }, { "id": "edge-75", "source": "input", "target": "image_encoder.patches", "sourcePort": null, "targetPort": null }, { "id": "edge-76", "source": "input", "target": "cat@image_encoder#0", "sourcePort": null, "targetPort": null }, { "id": "edge-77", "source": "input", "target": "image_encoder", "sourcePort": null, "targetPort": null }, { "id": "edge-78", "source": "input", "target": "text_encoder.tokens", "sourcePort": null, "targetPort": null }, { "id": "edge-79", "source": "input", "target": "text_encoder", "sourcePort": null, "targetPort": null }, { "id": "edge-80", "source": "python@image_encoder#1", "target": "mul@similarity#0", "sourcePort": null, "targetPort": null }, { "id": "edge-81", "source": "image_encoder", "target": "similarity", "sourcePort": null, "targetPort": null }, { "id": "edge-82", "source": "python@text_encoder#0", "target": "mul@similarity#0", "sourcePort": null, "targetPort": null }, { "id": "edge-83", "source": "text_encoder", "target": "similarity", "sourcePort": null, "targetPort": null }, { "id": "edge-84", "source": "mul@similarity#0", "target": "output", "sourcePort": null, "targetPort": null }, { "id": "edge-85", "source": "similarity", "target": "output", "sourcePort": null, "targetPort": null } ], "steps": [ { "id": "image", "target": "image_encoder.patches", "title": "Encode the image independently", "note": "Four 4 × 4 RGB patches become width-eight tokens. A class token, learned positions and one bidirectional Transformer produce a pooled image representation." }, { "id": "text", "target": "text_encoder.tokens", "title": "Encode the caption with causal attention", "note": "A separate text Transformer processes four token IDs with a causal mask. The last position stands for EOS and pools the preceding text. Neither encoder reads the other modality." }, { "id": "compare", "target": "similarity", "title": "Compare every image with every text", "note": "Each encoder linearly projects to four features and L2-normalizes the result. The similarity block computes all nine pairwise cosines and multiplies them by exp(log_scale), the learned inverse temperature." }, { "id": "objective", "target": "image_encoder.project", "title": "Train the paired diagonal in both directions", "note": "The local objective averages image-to-text and text-to-image cross entropy. Diagonal targets identify supplied pairs; batch negatives supply alternatives. Gradients reach both projections and both encoders. The panel exposes pairing and temperature effects." } ], "milestone": { "name": "CLIP", "year": 2021, "kind": "Training method", "fidelity": "One width-eight, two-head block per encoder. Four image patches plus a class token; four text positions with fixed final EOS pooling. Linear projections into four shared features and learned log-temperature.", "papers": [ { "title": "Learning Transferable Visual Models From Natural Language Supervision", "url": "https://arxiv.org/abs/2103.00020" } ] }, "experiment": { "kind": "matrices", "title": "Pair the batch in both directions.", "description": "Recorded similarities for three synthetic image/token pairs. Compare pairing order and temperature using the same untrained encoder weights.", "controlLabel": "Contrastive example", "cases": [ { "id": "paired", "label": "Original three pairs", "target": "similarity", "note": "Rows are images; columns are texts in the displayed order. Diagonal targets encode the supplied pairing, not model knowledge. Temperature rescales the same cosines before both softmax directions. These random encoders have no learned image–text alignment.", "matrices": [ { "label": "Cosine similarity · images × texts", "values": [ [ 0.24130594050529094, 0.182131262337695, 0.2244961837149168 ], [ 0.25702144291848616, 0.1939775501999552, 0.24462591414709337 ], [ 0.27740473306405133, 0.22011657756359881, 0.2607156290304313 ] ] }, { "label": "Image → text probabilities", "values": [ [ 0.45127897977842774, 0.19378187744515277, 0.3549391427764196 ], [ 0.4456271543759962, 0.18106491019248053, 0.37330793543152335 ], [ 0.44862946427857536, 0.19790657561373604, 0.3534639601076885 ] ] }, { "label": "Text → image probabilities", "values": [ [ 0.25467901436684187, 0.31878317273658335, 0.42653781289657483 ], [ 0.2560851290310808, 0.3033061247599923, 0.440608746208927 ], [ 0.24932213176686138, 0.332391449734795, 0.4182864184983435 ] ] } ], "vectors": [ { "label": "Text column order · original index", "values": [ 0, 1, 2 ] }, { "label": "Target column per image", "values": [ 0, 1, 2 ] } ], "metrics": [ { "label": "Temperature", "value": 0.07 }, { "label": "Symmetric contrastive loss", "value": 1.1628159711594945 }, { "label": "Mean image-row entropy · nats", "value": 1.0433215197908041 } ] }, { "id": "shuffled", "label": "Reverse only the text batch", "target": "similarity", "note": "Rows are images; columns are texts in the displayed order. Diagonal targets encode the supplied pairing, not model knowledge. Only text order changes; diagonal targets now ask for different pairings. Temperature rescales the same cosines before both softmax directions. These random encoders have no learned image–text alignment.", "matrices": [ { "label": "Cosine similarity · images × texts", "values": [ [ 0.2244961837149168, 0.182131262337695, 0.24130594050529094 ], [ 0.24462591414709337, 0.1939775501999552, 0.25702144291848616 ], [ 0.2607156290304313, 0.22011657756359881, 0.27740473306405133 ] ] }, { "label": "Image → text probabilities", "values": [ [ 0.3549391427764196, 0.19378187744515277, 0.45127897977842774 ], [ 0.37330793543152335, 0.18106491019248053, 0.4456271543759962 ], [ 0.3534639601076885, 0.19790657561373604, 0.44862946427857536 ] ] }, { "label": "Text → image probabilities", "values": [ [ 0.24932213176686138, 0.332391449734795, 0.4182864184983435 ], [ 0.2560851290310808, 0.3033061247599923, 0.440608746208927 ], [ 0.25467901436684187, 0.31878317273658335, 0.42653781289657483 ] ] } ], "vectors": [ { "label": "Text column order · original index", "values": [ 2, 1, 0 ] }, { "label": "Target column per image", "values": [ 0, 1, 2 ] } ], "metrics": [ { "label": "Temperature", "value": 0.07 }, { "label": "Symmetric contrastive loss", "value": 1.1633905080964186 }, { "label": "Mean image-row entropy · nats", "value": 1.0433215197908041 } ] }, { "id": "warm", "label": "Higher temperature · 0.5", "target": "similarity", "note": "Rows are images; columns are texts in the displayed order. Diagonal targets encode the supplied pairing, not model knowledge. Temperature rescales the same cosines before both softmax directions. These random encoders have no learned image–text alignment.", "matrices": [ { "label": "Cosine similarity · images × texts", "values": [ [ 0.24130594050529094, 0.182131262337695, 0.2244961837149168 ], [ 0.25702144291848616, 0.1939775501999552, 0.24462591414709337 ], [ 0.27740473306405133, 0.22011657756359881, 0.2607156290304313 ] ] }, { "label": "Image → text probabilities", "values": [ [ 0.35022283207324895, 0.31113293314865104, 0.3386442347781001 ], [ 0.3500112315203492, 0.3085480110383699, 0.34144075744128105 ], [ 0.34978285328475794, 0.31191672082791116, 0.338300425887331 ] ] }, { "label": "Text → image probabilities", "values": [ [ 0.32187481772237264, 0.33215233696033675, 0.3459728453172905 ], [ 0.3222784975760328, 0.3300052775021804, 0.3477162249217869 ], [ 0.32090284473636704, 0.3340858092190802, 0.34501134604455264 ] ] } ], "vectors": [ { "label": "Text column order · original index", "values": [ 0, 1, 2 ] }, { "label": "Target column per image", "values": [ 0, 1, 2 ] } ], "metrics": [ { "label": "Temperature", "value": 0.5 }, { "label": "Symmetric contrastive loss", "value": 1.1025502683648116 }, { "label": "Mean image-row entropy · nats", "value": 1.097341604704597 } ] }, { "id": "sharp", "label": "Lower temperature · 0.03", "target": "similarity", "note": "Rows are images; columns are texts in the displayed order. Diagonal targets encode the supplied pairing, not model knowledge. Temperature rescales the same cosines before both softmax directions. These random encoders have no learned image–text alignment.", "matrices": [ { "label": "Cosine similarity · images × texts", "values": [ [ 0.24130594050529094, 0.182131262337695, 0.2244961837149168 ], [ 0.25702144291848616, 0.1939775501999552, 0.24462591414709337 ], [ 0.27740473306405133, 0.22011657756359881, 0.2607156290304313 ] ] }, { "label": "Image → text probabilities", "values": [ [ 0.5847496825530024, 0.08134461128354552, 0.3339057061634521 ], [ 0.5605953862021676, 0.06854814314028632, 0.37085647065754607 ], [ 0.5809011347993279, 0.08605403347400184, 0.33304483172667043 ] ] }, { "label": "Text → image probabilities", "values": [ [ 0.1661244241701628, 0.28050376048118597, 0.5533718153486512 ], [ 0.16579715572525291, 0.24607622601278004, 0.5881266182619671 ], [ 0.1587133537787162, 0.3104710881506701, 0.5308155580706139 ] ] } ], "vectors": [ { "label": "Text column order · original index", "values": [ 0, 1, 2 ] }, { "label": "Target column per image", "values": [ 0, 1, 2 ] } ], "metrics": [ { "label": "Temperature", "value": 0.03 }, { "label": "Symmetric contrastive loss", "value": 1.3577902282348666 }, { "label": "Mean image-row entropy · nats", "value": 0.8843119450267872 } ] } ] }, "revision": "f82eeaabb5d24e21", "files": { "graph.tensorviz.json": { "url": "/models/clip/f82eeaabb5d24e21/graph.tensorviz.json", "sha256": "802c37778dc072c595e0cf317f2a621b5971685bbf7dee51fffb348be3cbe3ee", "bytes": 146397 }, "model.py": { "url": "/models/clip/f82eeaabb5d24e21/model.py", "sha256": "a7e72c3bbad2e899038b77cab7a58e4115cbe191500180e55a0058316f782711", "bytes": 8636 }, "run.py": { "url": "/models/clip/f82eeaabb5d24e21/run.py", "sha256": "596c95cf7cbe76960a56ce5efe434b5bdbb56da01c3e7ad4ea0e8a49eb9c97cd", "bytes": 688 }, "requirements.txt": { "url": "/models/clip/f82eeaabb5d24e21/requirements.txt", "sha256": "edb86016b42cb3cc6cdc0c490aab929d3bb098ced4515394f0b55766033e0daf", "bytes": 91 }, "preview.svg": { "url": "/models/clip/f82eeaabb5d24e21/preview.svg", "sha256": "69f00f2976420f3ec2262d136dcb826673f5008a45f17c449f751d6b8290c96e", "bytes": 1996 } } }