{
  "schemaVersion": 1,
  "lessonId": "pretraining-01",
  "contentHash": "e44d399501edc9669f44321bf1190105f65df97b7153323d41a087617b8458b3",
  "rendererHash": "a1bd5b648f4731ae5d5f0a24811c2540d1a0a132149ce589d17c4113509cbedf",
  "kind": "executed",
  "generatedAt": "2026-10-20T10:47:20.818351+00:00",
  "provenance": "recorded_cpu_computation",
  "environment": {
    "backend": "cpu",
    "deviceCount": 1,
    "jax": "0.9.2",
    "numpy": "2.4.4",
    "matplotlib": "3.10.8"
  },
  "stdout": "MLM initial/final/changed-mask: 1.0986123085021973 0.010633035562932491 0.010457751341164112\n",
  "data": {
    "panels": [
      {
        "kind": "line",
        "xlabel": "completed parameter updates before measurement",
        "ylabel": "selected-token cross-entropy (nats)",
        "series": [
          {
            "label": "recorded CPU training loss",
            "x": [
              0,
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8,
              9,
              10,
              11,
              12,
              13,
              14,
              15,
              16,
              17,
              18,
              19,
              20,
              21,
              22,
              23,
              24,
              25,
              26,
              27,
              28,
              29,
              30,
              31,
              32,
              33,
              34,
              35,
              36,
              37,
              38,
              39,
              40,
              41,
              42,
              43,
              44,
              45,
              46,
              47,
              48,
              49,
              50,
              51,
              52,
              53,
              54,
              55,
              56,
              57,
              58,
              59,
              60,
              61,
              62,
              63,
              64,
              65,
              66,
              67,
              68,
              69,
              70,
              71,
              72,
              73,
              74,
              75,
              76,
              77,
              78,
              79,
              80,
              81,
              82,
              83,
              84,
              85,
              86,
              87,
              88,
              89,
              90,
              91,
              92,
              93,
              94,
              95,
              96,
              97,
              98,
              99
            ],
            "y": [
              1.0986123085021973,
              1.0859612226486206,
              1.0732711553573608,
              1.0600903034210205,
              1.0459909439086914,
              1.0305562019348145,
              1.0133756399154663,
              0.9940430521965027,
              0.9721622467041016,
              0.9473583698272705,
              0.9192975759506226,
              0.8877142667770386,
              0.8524455428123474,
              0.8134702444076538,
              0.7709471583366394,
              0.7252441644668579,
              0.6769482493400574,
              0.626849889755249,
              0.575894832611084,
              0.5251108407974243,
              0.47551974654197693,
              0.4280504584312439,
              0.3834689259529114,
              0.34233206510543823,
              0.3049706816673279,
              0.2714996039867401,
              0.2418486773967743,
              0.21580862998962402,
              0.19307923316955566,
              0.17331385612487793,
              0.15615539252758026,
              0.14126010239124298,
              0.12831251323223114,
              0.1170317530632019,
              0.10717347264289856,
              0.09852820634841919,
              0.09091778099536896,
              0.08419191837310791,
              0.07822415232658386,
              0.07290790975093842,
              0.06815360486507416,
              0.0638858824968338,
              0.06004087254405022,
              0.05656450241804123,
              0.05341097712516785,
              0.05054116249084473,
              0.04792151600122452,
              0.045523300766944885,
              0.04332173615694046,
              0.04129539430141449,
              0.03942573815584183,
              0.037696462124586105,
              0.03609351068735123,
              0.034604381769895554,
              0.033218126744031906,
              0.03192529082298279,
              0.030717238783836365,
              0.029586464166641235,
              0.02852615714073181,
              0.027530377730727196,
              0.026593750342726707,
              0.02571152150630951,
              0.024879276752471924,
              0.0240931399166584,
              0.02334965579211712,
              0.022645581513643265,
              0.021978085860610008,
              0.02134447917342186,
              0.02074245922267437,
              0.020169783383607864,
              0.019624479115009308,
              0.01910475641489029,
              0.01860894076526165,
              0.018135471269488335,
              0.01768305152654648,
              0.017250265926122665,
              0.01683596521615982,
              0.016439005732536316,
              0.016058500856161118,
              0.0156934205442667,
              0.015342840924859047,
              0.015005992725491524,
              0.014682109467685223,
              0.014370575547218323,
              0.014070659875869751,
              0.013781705871224403,
              0.013503290712833405,
              0.013234758749604225,
              0.012975722551345825,
              0.012725567445158958,
              0.012483901344239712,
              0.012250381521880627,
              0.012024618685245514,
              0.011806267313659191,
              0.011594897136092186,
              0.011390200816094875,
              0.011192025616765022,
              0.01099994033575058,
              0.01081363670527935,
              0.010633035562932491
            ]
          }
        ],
        "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99
        ]
      },
      {
        "kind": "heatmap",
        "values": [
          [
            0.9895161390304565,
            0.0066155861131846905,
            0.003868206636980176
          ],
          [
            0.005779723171144724,
            0.989284098148346,
            0.004936168901622295
          ],
          [
            0.004057474434375763,
            0.0059524644166231155,
            0.9899901151657104
          ]
        ],
        "rows": [
          "target 0",
          "target 1",
          "target 2"
        ],
        "columns": [
          "class 0",
          "class 1",
          "class 2"
        ],
        "unit": "masked-token probability",
        "xlabel": "predicted vocabulary class",
        "ylabel": "masked example",
        "title": "Final predictions at the selected position"
      }
    ]
  },
  "diagram": {
    "title": "Keep clean targets out of encoder inputs",
    "prediction": "If all visible tokens reveal the same class, what limit does that place on the experiment?",
    "reading": "Clean tokens fork into labels and a corrupted-input path. Only corrupted inputs enter the encoder. Selected logits meet clean targets at the loss. This makes supervision available without leaking the hidden answer directly into the input.",
    "alt": "Clean tokens fork into labels and a corrupted-input path. Only corrupted inputs enter the encoder. Selected logits meet clean targets at the loss. This makes supervision available without leaking the hidden answer directly into the input.",
    "scope": "Architecture and dataflow mechanism diagram.",
    "height": 6,
    "panel": {
      "kind": "graph",
      "nodes": [
        {
          "id": "clean",
          "x": 5,
          "y": 1,
          "label": "Clean token sequence",
          "tone": "data",
          "w": 5,
          "h": 0.85
        },
        {
          "id": "corrupt",
          "x": 2.5,
          "y": 3,
          "label": "Corrupt selected inputs",
          "tone": "data",
          "w": 4,
          "h": 0.85
        },
        {
          "id": "labels",
          "x": 7.5,
          "y": 3,
          "label": "Retain clean labels",
          "tone": "fixed",
          "w": 4,
          "h": 0.85
        },
        {
          "id": "enc",
          "x": 2.5,
          "y": 5,
          "label": "Bidirectional encoder\n\u2192 selected logits",
          "tone": "data",
          "w": 4,
          "h": 1
        },
        {
          "id": "loss",
          "x": 5,
          "y": 7,
          "label": "Selected-position loss",
          "tone": "result",
          "w": 5,
          "h": 0.85
        }
      ],
      "edges": [
        {
          "from": "clean",
          "to": "corrupt"
        },
        {
          "from": "clean",
          "to": "labels"
        },
        {
          "from": "corrupt",
          "to": "enc"
        },
        {
          "from": "enc",
          "to": "loss"
        },
        {
          "from": "labels",
          "to": "loss"
        }
      ],
      "height": 8,
      "width": 10
    }
  }
}