{
  "schemaVersion": 1,
  "lessonId": "posttraining-04",
  "contentHash": "f95b5b63c839a7fb666de4e7d7d05613900eb9c24c3e422b20aed70502400f42",
  "rendererHash": "a1bd5b648f4731ae5d5f0a24811c2540d1a0a132149ce589d17c4113509cbedf",
  "kind": "executed",
  "generatedAt": "2026-10-20T10:47:32.867260+00:00",
  "provenance": "recorded_cpu_computation",
  "environment": {
    "backend": "cpu",
    "deviceCount": 1,
    "jax": "0.9.2",
    "numpy": "2.4.4",
    "matplotlib": "3.10.8"
  },
  "stdout": "Reward initial/final and final KL: 0.28191882371902466 1.9185447692871094 0.7824773192405701\nSynthetic preference bandit with actual sampled PPO updates; synthetic preference target or language-generation claim.\n",
  "data": {
    "panels": [
      {
        "kind": "line",
        "xlabel": "completed PPO rollout batches",
        "ylabel": "expected learned reward",
        "series": [
          {
            "label": "exact policy expectation",
            "x": [
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8,
              9,
              10,
              11,
              12,
              13,
              14,
              15,
              16,
              17,
              18,
              19,
              20,
              21,
              22,
              23,
              24,
              25,
              26,
              27,
              28,
              29,
              30,
              31,
              32,
              33,
              34,
              35,
              36,
              37,
              38,
              39,
              40
            ],
            "y": [
              0.6096429824829102,
              0.8819129467010498,
              1.124735951423645,
              1.3022879362106323,
              1.4446861743927002,
              1.5519723892211914,
              1.6030393838882446,
              1.6685360670089722,
              1.6937657594680786,
              1.714634895324707,
              1.7571830749511719,
              1.7731627225875854,
              1.7904469966888428,
              1.8034006357192993,
              1.8156886100769043,
              1.8244140148162842,
              1.824958086013794,
              1.8331801891326904,
              1.8488855361938477,
              1.8570934534072876,
              1.8618234395980835,
              1.8695471286773682,
              1.8747035264968872,
              1.8820245265960693,
              1.8839912414550781,
              1.89113187789917,
              1.8966443538665771,
              1.8979942798614502,
              1.9029959440231323,
              1.9053056240081787,
              1.9054564237594604,
              1.90715754032135,
              1.9084452390670776,
              1.9080511331558228,
              1.9111661911010742,
              1.912423849105835,
              1.9163932800292969,
              1.9176057577133179,
              1.918319582939148,
              1.9185447692871094
            ]
          }
        ],
        "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40
        ]
      },
      {
        "kind": "line",
        "xlabel": "completed PPO rollout batches",
        "ylabel": "KL to fixed reference (nats)",
        "series": [
          {
            "label": "exact categorical KL",
            "x": [
              1,
              2,
              3,
              4,
              5,
              6,
              7,
              8,
              9,
              10,
              11,
              12,
              13,
              14,
              15,
              16,
              17,
              18,
              19,
              20,
              21,
              22,
              23,
              24,
              25,
              26,
              27,
              28,
              29,
              30,
              31,
              32,
              33,
              34,
              35,
              36,
              37,
              38,
              39,
              40
            ],
            "y": [
              0.020994842052459717,
              0.07355361431837082,
              0.15258058905601501,
              0.23269957304000854,
              0.3141707181930542,
              0.386445015668869,
              0.4296932816505432,
              0.48545777797698975,
              0.5104964971542358,
              0.5317287445068359,
              0.5729994773864746,
              0.5898757576942444,
              0.6092323064804077,
              0.6257614493370056,
              0.6397278308868408,
              0.6502091288566589,
              0.651305615901947,
              0.6603918671607971,
              0.6801056861877441,
              0.6910279989242554,
              0.6976882815361023,
              0.7091653347015381,
              0.7167686223983765,
              0.7257503271102905,
              0.7284179925918579,
              0.7390725612640381,
              0.7472420334815979,
              0.7497239708900452,
              0.7566220760345459,
              0.7601989507675171,
              0.7606008052825928,
              0.7631310820579529,
              0.7655646204948425,
              0.7649880051612854,
              0.7695096135139465,
              0.7719148993492126,
              0.7781571745872498,
              0.780514121055603,
              0.7819498777389526,
              0.7824773192405701
            ]
          }
        ],
        "x": [
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19,
          20,
          21,
          22,
          23,
          24,
          25,
          26,
          27,
          28,
          29,
          30,
          31,
          32,
          33,
          34,
          35,
          36,
          37,
          38,
          39,
          40
        ]
      },
      {
        "kind": "bar",
        "x": [
          0,
          1,
          2
        ],
        "labels": [
          "action 0",
          "action 1",
          "action 2"
        ],
        "series": [
          {
            "label": "fixed reference",
            "y": [
              0.4017595648765564,
              0.32893288135528564,
              0.2693074941635132
            ]
          },
          {
            "label": "trained policy",
            "y": [
              0.9739989638328552,
              0.018571145832538605,
              0.007429859135299921
            ]
          }
        ],
        "xlabel": "terminal response action",
        "ylabel": "probability",
        "title": "How increased reward redistributes action probability"
      }
    ]
  },
  "diagram": {
    "title": "Different objects have different update lifetimes",
    "prediction": "Which frozen object refreshes between rollout batches: old policy probabilities or the reference policy?",
    "reading": "Current parameters change within PPO updates. Old probabilities stay fixed for that rollout batch and refresh next rollout. The reference and reward remain fixed during policy training. This categorical fixture has an exact baseline, so no learned critic is shown.",
    "alt": "Current parameters change within PPO updates. Old probabilities stay fixed for that rollout batch and refresh next rollout. The reference and reward remain fixed during policy training. This categorical fixture has an exact baseline, so no learned critic is shown.",
    "scope": "Architecture and dataflow mechanism diagram.",
    "height": 5.2,
    "panel": {
      "kind": "table",
      "headers": [
        "Object",
        "Purpose",
        "Changes when"
      ],
      "rows": [
        [
          "Current policy",
          "new probabilities",
          "inner updates"
        ],
        [
          "Old probabilities",
          "ratio denominator",
          "new rollout"
        ],
        [
          "Reference",
          "KL comparison",
          "fixed"
        ],
        [
          "Reward model",
          "score actions",
          "fitted earlier"
        ],
        [
          "Exact baseline",
          "center advantages",
          "rollout calculation"
        ]
      ]
    }
  }
}