{
  "schemaVersion": 1,
  "lessonId": "rl-03",
  "contentHash": "d6bbf2419915a048da0fe938b65db4c4792e2070f4c62e6f1714251e0ef5c702",
  "rendererHash": "a1bd5b648f4731ae5d5f0a24811c2540d1a0a132149ce589d17c4113509cbedf",
  "kind": "executed",
  "generatedAt": "2026-10-20T10:46:25.846602+00:00",
  "provenance": "recorded_cpu_computation",
  "environment": {
    "backend": "cpu",
    "deviceCount": 1,
    "jax": "0.9.2",
    "numpy": "2.4.4",
    "matplotlib": "3.10.8"
  },
  "stdout": "REINFORCE initial/final exact return: 0.46687505 0.9521624\nPPO initial/final exact return: 0.46687505 0.97578394\n",
  "data": {
    "kind": "line",
    "x": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40
    ],
    "xlabel": "fresh rollout batches",
    "ylabel": "exact expected episode return",
    "series": [
      {
        "label": "REINFORCE: 1 epoch/batch",
        "y": [
          0.4668750464916229,
          0.5134358406066895,
          0.5576684474945068,
          0.6021765470504761,
          0.645237386226654,
          0.6809811592102051,
          0.7145836353302002,
          0.7452755570411682,
          0.7698182463645935,
          0.7921379804611206,
          0.8098292350769043,
          0.8264045715332031,
          0.8420286178588867,
          0.8558729290962219,
          0.8680567145347595,
          0.8771833181381226,
          0.8842524290084839,
          0.8916422724723816,
          0.897716224193573,
          0.9027289152145386,
          0.9080594778060913,
          0.9099045991897583,
          0.9153724908828735,
          0.9198133945465088,
          0.9227747917175293,
          0.9263951778411865,
          0.9294171333312988,
          0.9332233667373657,
          0.93524169921875,
          0.9368733167648315,
          0.9385892748832703,
          0.9406171441078186,
          0.9416252374649048,
          0.9432613849639893,
          0.9447624683380127,
          0.9459059238433838,
          0.9471801519393921,
          0.9490498900413513,
          0.9501066207885742,
          0.9515848755836487,
          0.9521623849868774
        ]
      },
      {
        "label": "clipped updates: 3 epochs/batch",
        "y": [
          0.4668750464916229,
          0.6031153202056885,
          0.713788628578186,
          0.7927064299583435,
          0.8484611511230469,
          0.875048041343689,
          0.8989757299423218,
          0.9124115705490112,
          0.9207390546798706,
          0.9291005730628967,
          0.9348503351211548,
          0.939469575881958,
          0.9430556297302246,
          0.9477962255477905,
          0.9515244364738464,
          0.9562485814094543,
          0.9576464891433716,
          0.9603234529495239,
          0.9615710973739624,
          0.9622180461883545,
          0.9630686044692993,
          0.9631805419921875,
          0.9641537666320801,
          0.9665656685829163,
          0.9670358896255493,
          0.9686336517333984,
          0.96900475025177,
          0.9710692167282104,
          0.9714295864105225,
          0.9714440703392029,
          0.9720673561096191,
          0.9726168513298035,
          0.9731839895248413,
          0.9737018942832947,
          0.9741728901863098,
          0.9740797281265259,
          0.9746109247207642,
          0.9751652479171753,
          0.9751903414726257,
          0.9755208492279053,
          0.9757839441299438
        ]
      }
    ]
  }
}