{
  "schemaVersion": 1,
  "lessonId": "rl-05",
  "contentHash": "1c3970b69463953ce5252283cc06a983e6057a82139e020733151dd029d8f813",
  "rendererHash": "17e3bbad72ee4c43425845aaee76b86b85e84750551278be03d8c16842f4dc63",
  "kind": "executed",
  "generatedAt": "2026-10-27T17:11:06.596927+00:00",
  "provenance": "recorded_cpu_computation",
  "environment": {
    "backend": "cpu",
    "deviceCount": 1,
    "jax": "0.9.2",
    "numpy": "2.4.4",
    "matplotlib": "3.10.8"
  },
  "stdout": "initial / final masked value MSE: 0.3257838487625122 0.13587723672389984\nfitted V(position 0..3): [0.1953 0.4592 0.8403 0.    ]\ninitial / final masked value MSE: 0.3257838487625122 0.13587723672389984\nfitted V(position 0..3): [0.1953 0.4592 0.8403 0.    ]\nGAE shape: (8, 512) all finite: True\n",
  "data": {
    "kind": "line",
    "x": [
      0,
      1,
      2,
      3,
      4,
      5,
      6,
      7,
      8,
      9,
      10,
      11,
      12,
      13,
      14,
      15,
      16,
      17,
      18,
      19,
      20,
      21,
      22,
      23,
      24,
      25,
      26,
      27,
      28,
      29,
      30,
      31,
      32,
      33,
      34,
      35,
      36,
      37,
      38,
      39,
      40,
      41,
      42,
      43,
      44,
      45,
      46,
      47,
      48,
      49,
      50,
      51,
      52,
      53,
      54,
      55,
      56,
      57,
      58,
      59,
      60,
      61,
      62,
      63,
      64,
      65,
      66,
      67,
      68,
      69,
      70,
      71,
      72,
      73,
      74,
      75,
      76,
      77,
      78,
      79,
      80,
      81,
      82,
      83,
      84,
      85,
      86,
      87,
      88,
      89,
      90,
      91,
      92,
      93,
      94,
      95,
      96,
      97,
      98,
      99,
      100,
      101,
      102,
      103,
      104,
      105,
      106,
      107,
      108,
      109,
      110,
      111,
      112,
      113,
      114,
      115,
      116,
      117,
      118,
      119,
      120,
      121,
      122,
      123,
      124,
      125,
      126,
      127,
      128,
      129,
      130,
      131,
      132,
      133,
      134,
      135,
      136,
      137,
      138,
      139,
      140,
      141,
      142,
      143,
      144,
      145,
      146,
      147,
      148,
      149,
      150,
      151,
      152,
      153,
      154,
      155,
      156,
      157,
      158,
      159,
      160,
      161,
      162,
      163,
      164,
      165,
      166,
      167,
      168,
      169,
      170,
      171,
      172,
      173,
      174,
      175,
      176,
      177,
      178,
      179,
      180,
      181,
      182,
      183,
      184,
      185,
      186,
      187,
      188,
      189,
      190,
      191,
      192,
      193,
      194,
      195,
      196,
      197,
      198,
      199,
      200,
      201,
      202,
      203,
      204,
      205,
      206,
      207,
      208,
      209,
      210,
      211,
      212,
      213,
      214,
      215,
      216,
      217,
      218,
      219,
      220,
      221,
      222,
      223,
      224,
      225,
      226,
      227,
      228,
      229,
      230,
      231,
      232,
      233,
      234,
      235,
      236,
      237,
      238,
      239,
      240,
      241,
      242,
      243,
      244,
      245,
      246,
      247,
      248,
      249,
      250
    ],
    "xlabel": "gradient updates",
    "ylabel": "masked value prediction MSE",
    "series": [
      {
        "label": "sampled rollout, seed 313",
        "y": [
          0.3257838487625122,
          0.3079860508441925,
          0.2922928035259247,
          0.27839386463165283,
          0.2660332918167114,
          0.25499793887138367,
          0.24510973691940308,
          0.23621949553489685,
          0.22820177674293518,
          0.2209496647119522,
          0.21437259018421173,
          0.20839300751686096,
          0.2029445618391037,
          0.19796952605247498,
          0.19341835379600525,
          0.1892475187778473,
          0.18541908264160156,
          0.18189968168735504,
          0.17865991592407227,
          0.17567379772663116,
          0.1729184091091156,
          0.17037329077720642,
          0.1680198311805725,
          0.16584153473377228,
          0.16382378339767456,
          0.16195301711559296,
          0.16021741926670074,
          0.1586059033870697,
          0.15710888803005219,
          0.15571703016757965,
          0.15442240238189697,
          0.15321733057498932,
          0.1520952731370926,
          0.15105007588863373,
          0.15007564425468445,
          0.14916692674160004,
          0.1483195424079895,
          0.14752836525440216,
          0.1467897742986679,
          0.14610007405281067,
          0.14545579254627228,
          0.144853413105011,
          0.14429058134555817,
          0.14376400411128998,
          0.14327144622802734,
          0.14281095564365387,
          0.1423797458410263,
          0.1419760137796402,
          0.14159828424453735,
          0.14124441146850586,
          0.14091293513774872,
          0.14060261845588684,
          0.14031165838241577,
          0.14003890752792358,
          0.1397833228111267,
          0.1395440697669983,
          0.1393192857503891,
          0.1391087919473648,
          0.1389112025499344,
          0.13872583210468292,
          0.13855211436748505,
          0.13838902115821838,
          0.13823598623275757,
          0.13809257745742798,
          0.13795791566371918,
          0.1378314197063446,
          0.13771283626556396,
          0.13760150969028473,
          0.13749678432941437,
          0.13739854097366333,
          0.13730649650096893,
          0.13721995055675507,
          0.1371387094259262,
          0.13706232607364655,
          0.1369907259941101,
          0.13692344725131989,
          0.13686014711856842,
          0.13680076599121094,
          0.13674503564834595,
          0.13669274747371674,
          0.13664355874061584,
          0.13659727573394775,
          0.1365538090467453,
          0.13651305437088013,
          0.13647468388080597,
          0.13643872737884521,
          0.13640494644641876,
          0.13637298345565796,
          0.13634325563907623,
          0.13631519675254822,
          0.13628876209259033,
          0.13626404106616974,
          0.1362408548593521,
          0.13621889054775238,
          0.13619829714298248,
          0.1361789107322693,
          0.136160746216774,
          0.13614369928836823,
          0.13612760603427887,
          0.1361125111579895,
          0.13609853386878967,
          0.13608524203300476,
          0.13607275485992432,
          0.13606102764606476,
          0.13604995608329773,
          0.13603948056697845,
          0.13602976500988007,
          0.13602054119110107,
          0.13601195812225342,
          0.13600388169288635,
          0.13599620759487152,
          0.1359890103340149,
          0.13598231971263885,
          0.13597597181797028,
          0.1359701007604599,
          0.1359645277261734,
          0.13595925271511078,
          0.13595432043075562,
          0.1359497755765915,
          0.13594533503055573,
          0.13594120740890503,
          0.13593734800815582,
          0.13593371212482452,
          0.13593026995658875,
          0.13592717051506042,
          0.13592414557933807,
          0.13592129945755005,
          0.1359187215566635,
          0.13591621816158295,
          0.1359138786792755,
          0.1359117180109024,
          0.13590963184833527,
          0.13590766489505768,
          0.13590587675571442,
          0.13590414822101593,
          0.1359025090932846,
          0.13590100407600403,
          0.13589951395988464,
          0.13589821755886078,
          0.13589692115783691,
          0.1358957141637802,
          0.13589458167552948,
          0.1358935683965683,
          0.1358925849199295,
          0.1358916163444519,
          0.13589084148406982,
          0.13589002192020416,
          0.13588915765285492,
          0.13588851690292358,
          0.13588780164718628,
          0.13588717579841614,
          0.1358865350484848,
          0.13588596880435944,
          0.13588552176952362,
          0.13588492572307587,
          0.13588449358940125,
          0.13588403165340424,
          0.13588370382785797,
          0.13588324189186096,
          0.1358828991651535,
          0.13588251173496246,
          0.13588222861289978,
          0.13588184118270874,
          0.13588164746761322,
          0.13588134944438934,
          0.13588112592697144,
          0.13588084280490875,
          0.13588064908981323,
          0.13588044047355652,
          0.1358802765607834,
          0.13588008284568787,
          0.13587984442710876,
          0.1358797252178192,
          0.1358795464038849,
          0.13587947189807892,
          0.1358792930841446,
          0.13587914407253265,
          0.13587898015975952,
          0.13587887585163116,
          0.1358788162469864,
          0.13587869703769684,
          0.13587862253189087,
          0.13587850332260132,
          0.1358785480260849,
          0.13587838411331177,
          0.13587826490402222,
          0.13587826490402222,
          0.13587822020053864,
          0.13587816059589386,
          0.1358780413866043,
          0.13587790727615356,
          0.13587789237499237,
          0.1358778476715088,
          0.13587795197963715,
          0.13587786257266998,
          0.13587777316570282,
          0.13587771356105804,
          0.13587769865989685,
          0.13587763905525208,
          0.13587751984596252,
          0.1358775496482849,
          0.13587751984596252,
          0.13587763905525208,
          0.13587753474712372,
          0.13587751984596252,
          0.13587753474712372,
          0.13587747514247894,
          0.13587747514247894,
          0.13587740063667297,
          0.13587746024131775,
          0.13587737083435059,
          0.1358773112297058,
          0.1358773112297058,
          0.1358773112297058,
          0.13587728142738342,
          0.13587726652622223,
          0.13587728142738342,
          0.13587723672389984,
          0.13587723672389984,
          0.13587725162506104,
          0.13587723672389984,
          0.13587723672389984,
          0.13587726652622223,
          0.13587722182273865,
          0.13587728142738342,
          0.13587726652622223,
          0.1358773112297058,
          0.13587729632854462,
          0.13587726652622223,
          0.13587723672389984,
          0.1358773112297058,
          0.13587725162506104,
          0.13587728142738342,
          0.13587723672389984,
          0.13587720692157745,
          0.13587722182273865,
          0.13587722182273865,
          0.13587719202041626,
          0.13587719202041626,
          0.13587723672389984,
          0.13587720692157745,
          0.13587725162506104,
          0.13587719202041626,
          0.13587723672389984,
          0.13587719202041626,
          0.13587719202041626,
          0.13587722182273865,
          0.13587723672389984,
          0.13587722182273865,
          0.13587723672389984,
          0.13587723672389984
        ]
      }
    ]
  },
  "diagram": {
    "title": "A cutoff bootstraps value but stops reverse GAE",
    "prediction": "If the collector cutoff is the task\u2019s true finite deadline, which bootstrap term and advantage values change?",
    "reading": "Read the graph left to right as \\(t=0: s_0=0\\to s_1=1, r_0=1\\) then \\(t=1: s_1=1\\to s_2=3, r_1=2\\); the reverse arrow points from the second residual back to the first with coefficient \\(\\gamma\\lambda=0.25\\). For this diagram only, \\(\\gamma=\\lambda=0.5\\): continuing cutoff gives \\(\\delta_1=A_1=3\\), then \\(A_0=1.75\\); making it the finite deadline removes \\(\\gamma V(3)=2\\), giving \\(\\delta_1=1\\) and \\(A_0=1.25\\). The trace mask is zero in both cases, isolating reset episode \\(t=2, s_2=0\\). A true terminal also has bootstrap mask zero. Predict which number changes if the collector cutoff is reclassified as the task deadline: the bootstrap contribution changes, while the trace boundary stays closed. The critic-fit plot elsewhere uses gamma=0.9.",
    "alt": "Mechanism diagram for reverse GAE. At t=0, position 0 transitions to position 1 with reward 1, delta 1 and trace mask 1. At t=1, position 1 transitions to final pre-reset position 3 with reward 2. Continuing-cutoff bootstrap mask 1 gives delta and advantage 3; the trace mask is 0. The reverse arrow contributes 0.25 times 3 to A0, giving 1.75. A reset observation at position 0 is isolated from the prior episode. At a finite deadline or true terminal, bootstrap mask is 0, giving delta 1 and A0 1.25 for the finite-horizon case.",
    "scope": "Deterministic hand fixture with gamma=0.5 and lambda=0.5; the critic loss plot separately uses gamma=0.9 discounted targets.",
    "height": 7.2,
    "panel": {
      "kind": "graph",
      "width": 12,
      "height": 8,
      "nodes": [
        {
          "id": "t0",
          "x": 2.2,
          "y": 4.1,
          "w": 3.7,
          "h": 2,
          "tone": "data",
          "fontSize": 10,
          "label": "t=0: pos0 \u2192 pos1, r=1\nV(0)=0.5; V(1)=1\n\u03b4\u2080=1; m_boot=1; m_trace=1\nA\u2080=1 + 0.25 A\u2081\nA\u2080=1.75 cutoff / 1.25 finite"
        },
        {
          "id": "t1",
          "x": 7,
          "y": 4.1,
          "w": 4.5,
          "h": 2.5,
          "tone": "result",
          "fontSize": 10,
          "label": "t=1 collector cutoff\npos1 \u2192 final pre-reset pos3, r=2\nV(1)=1; V(3)=4\ncontinue: boot=1, \u03b4\u2081=A\u2081=3\nfinite: boot=0, \u03b4\u2081=A\u2081=1\nboth: trace=0"
        },
        {
          "id": "reset",
          "x": 10,
          "y": 1,
          "w": 3.8,
          "h": 1.5,
          "tone": "warning",
          "fontSize": 10,
          "label": "Next episode: t=2, obs pos0\nA\u2082=0 in the hand fixture\nNo A\u2082 \u2192 A\u2081: cutoff m_trace=0"
        },
        {
          "id": "terminal",
          "x": 7,
          "y": 7.2,
          "w": 4.2,
          "h": 1.5,
          "tone": "fixed",
          "fontSize": 10,
          "label": "True terminal alternative, t=1\nr=2, V(current)=1\nm_boot=0, \u03b4\u2081=A\u2081=1\nm_trace=0"
        }
      ],
      "edges": [
        {
          "from": "t1",
          "to": "t0",
          "exit": "left",
          "enter": "right",
          "label": "reverse scan: \u03b3\u03bb m_trace(t=0)=0.25",
          "labelAt": [
            4.55,
            2.8
          ]
        }
      ]
    }
  }
}
