{
  "schemaVersion": 1,
  "lessonId": "posttraining-05",
  "contentHash": "247b12d5db6bb478abfe606e2a31ed9fe445bee53ee43b39281b48654fdf44cc",
  "rendererHash": "a1bd5b648f4731ae5d5f0a24811c2540d1a0a132149ce589d17c4113509cbedf",
  "kind": "executed",
  "generatedAt": "2026-10-20T10:47:34.370265+00:00",
  "provenance": "recorded_cpu_computation",
  "environment": {
    "backend": "cpu",
    "deviceCount": 1,
    "jax": "0.9.2",
    "numpy": "2.4.4",
    "matplotlib": "3.10.8"
  },
  "stdout": "DPO initial/final: 0.6931471824645996 0.27227783203125 ; reference-corrected margins: [3.0890286  6.17805719 3.08902884]\n",
  "data": {
    "panels": [
      {
        "kind": "line",
        "xlabel": "completed parameter updates before measurement",
        "ylabel": "DPO preference loss (nats)",
        "series": [
          {
            "label": "recorded CPU training loss",
            "x": [
              0,
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8,
              9,
              10,
              11,
              12,
              13,
              14,
              15,
              16,
              17,
              18,
              19,
              20,
              21,
              22,
              23,
              24,
              25,
              26,
              27,
              28,
              29,
              30,
              31,
              32,
              33,
              34,
              35,
              36,
              37,
              38,
              39,
              40,
              41,
              42,
              43,
              44,
              45,
              46,
              47,
              48,
              49,
              50,
              51,
              52,
              53,
              54,
              55,
              56,
              57,
              58,
              59,
              60,
              61,
              62,
              63,
              64,
              65,
              66,
              67,
              68,
              69,
              70,
              71,
              72,
              73,
              74,
              75,
              76,
              77,
              78,
              79,
              80,
              81,
              82,
              83,
              84,
              85,
              86,
              87,
              88,
              89,
              90,
              91,
              92,
              93,
              94,
              95,
              96,
              97,
              98,
              99,
              100,
              101,
              102,
              103,
              104,
              105,
              106,
              107,
              108,
              109,
              110,
              111,
              112,
              113,
              114,
              115,
              116,
              117,
              118,
              119
            ],
            "y": [
              0.6931471824645996,
              0.6851832270622253,
              0.6773619055747986,
              0.6696807146072388,
              0.662136971950531,
              0.6547282934188843,
              0.6474520564079285,
              0.640305757522583,
              0.6332868337631226,
              0.6263930201530457,
              0.619621753692627,
              0.6129707098007202,
              0.6064374446868896,
              0.6000196933746338,
              0.5937151908874512,
              0.5875216722488403,
              0.5814367532730103,
              0.5754584670066833,
              0.5695845484733582,
              0.5638128519058228,
              0.5581413507461548,
              0.5525678992271423,
              0.5470906496047974,
              0.541707456111908,
              0.5364164710044861,
              0.5312157869338989,
              0.5261034965515137,
              0.5210778117179871,
              0.516136884689331,
              0.5112791061401367,
              0.5065025091171265,
              0.5018055438995361,
              0.4971866011619568,
              0.49264395236968994,
              0.48817604780197144,
              0.4837814271450043,
              0.47945839166641235,
              0.47520551085472107,
              0.4710213840007782,
              0.4669046401977539,
              0.4628537595272064,
              0.4588674008846283,
              0.45494431257247925,
              0.4510831832885742,
              0.4472825527191162,
              0.44354137778282166,
              0.43985843658447266,
              0.43623241782188416,
              0.4326622486114502,
              0.4291467070579529,
              0.4256848692893982,
              0.4222753942012787,
              0.41891735792160034,
              0.4156096875667572,
              0.41235142946243286,
              0.4091416001319885,
              0.40597909688949585,
              0.402863085269928,
              0.3997926115989685,
              0.3967667520046234,
              0.39378464221954346,
              0.39084547758102417,
              0.38794827461242676,
              0.3850923776626587,
              0.38227689266204834,
              0.37950098514556885,
              0.3767640292644501,
              0.3740651607513428,
              0.37140366435050964,
              0.3687788248062134,
              0.3661899268627167,
              0.3636363446712494,
              0.3611173629760742,
              0.3586323857307434,
              0.35618066787719727,
              0.3537616431713104,
              0.3513747453689575,
              0.34901928901672363,
              0.34669482707977295,
              0.34440064430236816,
              0.3421362340450287,
              0.33990105986595154,
              0.3376946449279785,
              0.3355163037776947,
              0.33336567878723145,
              0.33124229311943054,
              0.32914549112319946,
              0.32707494497299194,
              0.325030118227005,
              0.32301056385040283,
              0.3210158944129944,
              0.31904563307762146,
              0.31709927320480347,
              0.3151765465736389,
              0.31327688694000244,
              0.31140008568763733,
              0.3095456063747406,
              0.307713121175766,
              0.30590224266052246,
              0.3041125237941742,
              0.30234381556510925,
              0.3005955219268799,
              0.29886746406555176,
              0.29715925455093384,
              0.29547056555747986,
              0.29380103945732117,
              0.2921504080295563,
              0.2905183732509613,
              0.2889045476913452,
              0.28730878233909607,
              0.28573060035705566,
              0.28416985273361206,
              0.2826262414455414,
              0.28109949827194214,
              0.2795892655849457,
              0.2780953645706177,
              0.27661752700805664,
              0.2751554846763611,
              0.2737089991569519,
              0.27227783203125
            ]
          }
        ],
        "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40,
          41,
          42,
          43,
          44,
          45,
          46,
          47,
          48,
          49,
          50,
          51,
          52,
          53,
          54,
          55,
          56,
          57,
          58,
          59,
          60,
          61,
          62,
          63,
          64,
          65,
          66,
          67,
          68,
          69,
          70,
          71,
          72,
          73,
          74,
          75,
          76,
          77,
          78,
          79,
          80,
          81,
          82,
          83,
          84,
          85,
          86,
          87,
          88,
          89,
          90,
          91,
          92,
          93,
          94,
          95,
          96,
          97,
          98,
          99,
          100,
          101,
          102,
          103,
          104,
          105,
          106,
          107,
          108,
          109,
          110,
          111,
          112,
          113,
          114,
          115,
          116,
          117,
          118,
          119
        ]
      },
      {
        "kind": "bar",
        "x": [
          0,
          1,
          2
        ],
        "labels": [
          "0 preferred to 1",
          "0 preferred to 2",
          "1 preferred to 2"
        ],
        "series": [
          {
            "label": "final corrected margin",
            "y": [
              3.0890285968780518,
              6.1780571937561035,
              3.089028835296631
            ]
          }
        ],
        "xlabel": "preference pair",
        "ylabel": "reference-corrected log ratio",
        "title": "Relative preference changes behind the loss"
      }
    ]
  },
  "diagram": {
    "title": "DPO corrects a policy margin with a reference",
    "prediction": "What extra evidence would you inspect before claiming a DPO update makes chosen responses more likely?",
    "reading": "Both models score the same chosen/rejected pair. Their log-probability differences form a reference-corrected margin, which beta scales before the stable preference loss. The reference is frozen. There is no reward-model call or on-policy rollout in this step.",
    "alt": "Both models score the same chosen/rejected pair. Their log-probability differences form a reference-corrected margin, which beta scales before the stable preference loss. The reference is frozen. There is no reward-model call or on-policy rollout in this step.",
    "scope": "Architecture and dataflow mechanism diagram.",
    "height": 5.7,
    "panel": {
      "kind": "graph",
      "nodes": [
        {
          "id": "pairs",
          "x": 5,
          "y": 0.8,
          "label": "Chosen / rejected pair",
          "tone": "data",
          "w": 5,
          "h": 0.85
        },
        {
          "id": "policy",
          "x": 2.5,
          "y": 2.6,
          "label": "Trainable policy\nchosen \u2212 rejected log p",
          "tone": "state",
          "w": 4.4,
          "h": 1.1
        },
        {
          "id": "ref",
          "x": 7.5,
          "y": 2.6,
          "label": "Frozen reference\nchosen \u2212 rejected log p",
          "tone": "fixed",
          "w": 4.4,
          "h": 1.1
        },
        {
          "id": "margin",
          "x": 5,
          "y": 4.5,
          "label": "Policy difference \u2212 reference difference",
          "tone": "data",
          "w": 8,
          "h": 0.85
        },
        {
          "id": "loss",
          "x": 5,
          "y": 6.3,
          "label": "softplus(\u2212 beta \u00d7 margin)",
          "tone": "result",
          "w": 6,
          "h": 0.85
        }
      ],
      "edges": [
        {
          "from": "pairs",
          "to": "policy"
        },
        {
          "from": "pairs",
          "to": "ref"
        },
        {
          "from": "policy",
          "to": "margin"
        },
        {
          "from": "ref",
          "to": "margin"
        },
        {
          "from": "margin",
          "to": "loss"
        }
      ],
      "height": 7.3,
      "width": 10
    }
  }
}