{
 "name": "so101-block-stack-until-done",
 "columns": [
  {
   "key": "standing",
   "label": "Stacked",
   "better": "higher",
   "format": "pct",
   "description": "Layouts where the stack stands once the arm is cleared away"
  },
  {
   "key": "lifted",
   "label": "Lifted",
   "better": "higher",
   "format": "pct",
   "description": "Layouts where the jaws held the named block off the table"
  },
  {
   "key": "cost",
   "label": "Cost",
   "better": "lower",
   "format": ".3f",
   "description": "Mean model cost per trial, in US dollars"
  },
  {
   "key": "calls",
   "label": "Calls",
   "better": "lower",
   "format": ".0f",
   "description": "Mean model calls per trial"
  },
  {
   "key": "tokens_in",
   "label": "Tokens in",
   "better": "lower",
   "format": ",.0f",
   "description": "Mean prompt tokens per trial"
  },
  {
   "key": "tokens_out",
   "label": "Tokens out",
   "better": "lower",
   "format": ",.0f",
   "description": "Mean completion tokens per trial, reasoning included"
  },
  {
   "key": "minutes",
   "label": "Minutes",
   "better": "lower",
   "format": ".1f",
   "description": "Mean time per trial spent waiting on the model"
  }
 ],
 "info": {
  "provider": "Provider"
 },
 "rows": [
  {
   "name": "claude-opus-5-5",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.88,
     "lo": 0.78,
     "hi": 0.96
    },
    "lifted": {
     "value": 1.0,
     "lo": 1.0,
     "hi": 1.0
    },
    "cost": {
     "value": 0.237051952,
     "lo": 0.18755348040000003,
     "hi": 0.2925805771
    },
    "calls": {
     "value": 17.86,
     "lo": 15.06,
     "hi": 21.060999999999996
    },
    "tokens_in": {
     "value": 136590.24,
     "lo": 97057.98000000001,
     "hi": 182857.5825
    },
    "tokens_out": {
     "value": 2875.6,
     "lo": 2273.015,
     "hi": 3568.0434999999998
    },
    "minutes": {
     "value": 1.0065871613820394,
     "lo": 0.8137308594942094,
     "hi": 1.2328499240974586
    }
   },
   "info": {
    "provider": "anthropic"
   }
  },
  {
   "name": "pi0.5",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.82,
     "lo": 0.7195000000000005,
     "hi": 0.92
    },
    "lifted": {
     "value": 1.0,
     "lo": 1.0,
     "hi": 1.0
    }
   },
   "info": {
    "provider": "pi"
   }
  },
  {
   "name": "pi0",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.76,
     "lo": 0.64,
     "hi": 0.86
    },
    "lifted": {
     "value": 0.94,
     "lo": 0.86,
     "hi": 1.0
    }
   },
   "info": {
    "provider": "pi"
   }
  },
  {
   "name": "lawam",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.62,
     "lo": 0.49950000000000044,
     "hi": 0.74
    },
    "lifted": {
     "value": 0.9,
     "lo": 0.82,
     "hi": 0.98
    }
   },
   "info": {
    "provider": "rlinf"
   }
  },
  {
   "name": "claude-sonnet-5-5",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.52,
     "lo": 0.39950000000000047,
     "hi": 0.66
    },
    "lifted": {
     "value": 0.92,
     "lo": 0.84,
     "hi": 0.98
    },
    "cost": {
     "value": 0.23371134199999996,
     "lo": 0.17822986704999996,
     "hi": 0.2912328782
    },
    "calls": {
     "value": 29.0,
     "lo": 23.3775,
     "hi": 34.741
    },
    "tokens_in": {
     "value": 336394.34,
     "lo": 240106.94850000006,
     "hi": 443340.454
    },
    "tokens_out": {
     "value": 5743.24,
     "lo": 4382.6575,
     "hi": 7208.6505
    },
    "minutes": {
     "value": 1.621804824988047,
     "lo": 1.2625159583369892,
     "hi": 1.9942272004763286
    }
   },
   "info": {
    "provider": "anthropic"
   }
  },
  {
   "name": "vla-jepa",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.5,
     "lo": 0.36,
     "hi": 0.64
    },
    "lifted": {
     "value": 0.86,
     "lo": 0.74,
     "hi": 0.9404999999999994
    }
   },
   "info": {
    "provider": "ustc"
   }
  },
  {
   "name": "claude-haiku-5-5",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.34,
     "lo": 0.22,
     "hi": 0.48
    },
    "lifted": {
     "value": 0.86,
     "lo": 0.76,
     "hi": 0.94
    },
    "cost": {
     "value": 0.0200215248,
     "lo": 0.016406661985,
     "hi": 0.023900544914999995
    },
    "calls": {
     "value": 37.58,
     "lo": 31.819000000000003,
     "hi": 43.82
    },
    "tokens_in": {
     "value": 523772.98,
     "lo": 405062.88100000005,
     "hi": 659627.456
    },
    "tokens_out": {
     "value": 10333.46,
     "lo": 8719.401000000002,
     "hi": 12083.905999999999
    },
    "minutes": {
     "value": 1.315359795093536,
     "lo": 1.0895635158677899,
     "hi": 1.5529590081910292
    }
   },
   "info": {
    "provider": "anthropic"
   }
  },
  {
   "name": "deepseek-v4.1-flash",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.3,
     "lo": 0.18,
     "hi": 0.42049999999999954
    },
    "lifted": {
     "value": 0.9,
     "lo": 0.8,
     "hi": 0.98
    },
    "cost": {
     "value": 0.082053115544,
     "lo": 0.06781855336539999,
     "hi": 0.0976707645015
    },
    "calls": {
     "value": 61.88,
     "lo": 55.4995,
     "hi": 68.12
    },
    "tokens_in": {
     "value": 723513.72,
     "lo": 606358.0945,
     "hi": 848615.4935
    },
    "tokens_out": {
     "value": 76012.32,
     "lo": 63320.8835,
     "hi": 89936.2365
    },
    "minutes": {
     "value": 8.22877533340454,
     "lo": 6.563818119198085,
     "hi": 10.176058207744358
    }
   },
   "info": {
    "provider": "deepseek"
   }
  },
  {
   "name": "glm-5.3-flash",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.24,
     "lo": 0.12,
     "hi": 0.36
    },
    "lifted": {
     "value": 0.7,
     "lo": 0.58,
     "hi": 0.82
    },
    "cost": {
     "value": 0.03911766075,
     "lo": 0.0300025864607,
     "hi": 0.04930268183555
    },
    "calls": {
     "value": 50.2,
     "lo": 42.4,
     "hi": 57.6855
    },
    "tokens_in": {
     "value": 475918.6,
     "lo": 376361.2515,
     "hi": 578306.88
    },
    "tokens_out": {
     "value": 7002.3,
     "lo": 5860.795000000001,
     "hi": 8240.512
    },
    "minutes": {
     "value": 2.0868290885289507,
     "lo": 1.6441698789576689,
     "hi": 2.533948498580853
    }
   },
   "info": {
    "provider": "z-ai"
   }
  },
  {
   "name": "gpt-6-luna",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.16,
     "lo": 0.06,
     "hi": 0.26
    },
    "lifted": {
     "value": 0.34,
     "lo": 0.22,
     "hi": 0.48
    },
    "cost": {
     "value": 0.010555728000000002,
     "lo": 0.008166024595,
     "hi": 0.013428447077499999
    },
    "calls": {
     "value": 23.5,
     "lo": 19.979000000000003,
     "hi": 27.6405
    },
    "tokens_in": {
     "value": 127303.86,
     "lo": 100403.0705,
     "hi": 158643.13
    },
    "tokens_out": {
     "value": 2051.92,
     "lo": 1754.8215,
     "hi": 2394.8265
    },
    "minutes": {
     "value": 1.4961288495858511,
     "lo": 1.26279573452274,
     "hi": 1.7884148646414277
    }
   },
   "info": {
    "provider": "openai"
   }
  },
  {
   "name": "nemotron-3.5-super-vl",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.0,
     "lo": 0.0,
     "hi": 0.0
    },
    "lifted": {
     "value": 0.12,
     "lo": 0.04,
     "hi": 0.22
    },
    "cost": {
     "value": 0.0545902812,
     "lo": 0.04519968475000001,
     "hi": 0.06511021214500001
    },
    "calls": {
     "value": 51.6,
     "lo": 43.037,
     "hi": 60.864
    },
    "tokens_in": {
     "value": 636514.48,
     "lo": 495357.711,
     "hi": 792007.925
    },
    "tokens_out": {
     "value": 17840.58,
     "lo": 15322.6105,
     "hi": 20590.396
    },
    "minutes": {
     "value": 1.457367291053136,
     "lo": 1.2174425367097057,
     "hi": 1.7093066832304
    }
   },
   "info": {
    "provider": "nvidia"
   }
  },
  {
   "name": "smolvla",
   "kind": "model",
   "scores": {
    "standing": {
     "value": 0.0,
     "lo": 0.0,
     "hi": 0.0
    },
    "lifted": {
     "value": 0.16,
     "lo": 0.06,
     "hi": 0.26
    }
   },
   "info": {
    "provider": "huggingface"
   }
  }
 ],
 "notes": [
  "50 layouts, one trial each; 95% bootstrap interval over layouts. Cost in US dollars, calls, tokens and minutes waiting on the model are means per trial. Prompt version `8dd3b142780d`.",
  "The pi0.5, pi0, LaWAM, VLA-JEPA and SmolVLA rows are VLAs, each a LoRA fine-tune of the base model on the cell's [oracle demonstrations](https://huggingface.co/datasets/drwahl/so101_block_stack_sim): [pi0.5](https://huggingface.co/drwahl/pi05_so101_block_stack_sim), [pi0](https://huggingface.co/drwahl/pi0_so101_block_stack_sim), [LaWAM](https://huggingface.co/drwahl/lawam_so101_block_stack_sim), [VLA-JEPA](https://huggingface.co/drwahl/vla_jepa_so101_block_stack_sim), [SmolVLA](https://huggingface.co/drwahl/smolvla_so101_block_stack_sim). Each commands the joint targets directly and executes 10 actions of each chunk before predicting the next (VLA-JEPA 7, SmolVLA all 50), read back off a smooth cubic through 13 of its steps. pi0.5, pi0 and SmolVLA stitch their chunks with Real-Time Chunking."
 ],
 "updated": "2026-11-07"
}