{ "predictions_sha256": "19d9f17bdb4bc45426a12c24351ffb680f1403371f26858d224e534b7e31bdec", "scoring_implementation": "assessment/score_submission.py (standalone; repository scorer not present in this benchmark folder)", "total_predictions": 306, "excluded_example_ids": [ "s0019_02801c08_r_1715", "s0112_144a9b9a_r_825", "s0153_28d21458_l_1731", "s0296_4841aa44_r_866" ], "missing_ids": [], "extra_ids": [], "overall": { "n": 302, "correct": 146, "accuracy": 0.48344370860927155, "balanced_accuracy": 0.48403508771929826, "success_recall": 0.5733333333333334, "failure_recall": 0.39473684210526316, "auroc": 0.5073464912280702, "brier": 0.26164536423841067, "ece_10_equal_width_bins": 0.11745033112582792, "confusion_matrix": { "true_success_pred_success": 86, "true_success_pred_fail": 64, "true_fail_pred_success": 92, "true_fail_pred_fail": 60 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 2, "mean_probability": 0.275, "observed_success_rate": 0.5 }, { "lower": 0.3, "upper": 0.4, "count": 44, "mean_probability": 0.35886363636363633, "observed_success_rate": 0.5227272727272727 }, { "lower": 0.4, "upper": 0.5, "count": 78, "mean_probability": 0.4397435897435896, "observed_success_rate": 0.5128205128205128 }, { "lower": 0.5, "upper": 0.6, "count": 80, "mean_probability": 0.558625, "observed_success_rate": 0.425 }, { "lower": 0.6, "upper": 0.7, "count": 93, "mean_probability": 0.6345161290322583, "observed_success_rate": 0.5161290322580645 }, { "lower": 0.7, "upper": 0.8, "count": 5, "mean_probability": 0.718, "observed_success_rate": 0.8 } ], "strict_accuracy": 0.48344370860927155 }, "per_type": { "fail_at_decision": { "n": 106, "correct": 42, "accuracy": 0.39622641509433965, "balanced_accuracy": null, "success_recall": null, "failure_recall": 0.39622641509433965, "auroc": null, "brier": 0.2860415094339621, "ece_10_equal_width_bins": 0.5245283018867924, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 0, "true_fail_pred_success": 64, "true_fail_pred_fail": 42 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 15, "mean_probability": 0.35799999999999993, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 27, "mean_probability": 0.4344444444444445, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 28, "mean_probability": 0.5639285714285714, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 36, "mean_probability": 0.6308333333333334, "observed_success_rate": 0.0 } ] }, "fail_early": { "n": 46, "correct": 18, "accuracy": 0.391304347826087, "balanced_accuracy": null, "success_recall": null, "failure_recall": 0.391304347826087, "auroc": null, "brier": 0.2781434782608696, "ece_10_equal_width_bins": 0.5173913043478261, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 0, "true_fail_pred_success": 28, "true_fail_pred_fail": 18 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.28, "observed_success_rate": 0.0 }, { "lower": 0.3, "upper": 0.4, "count": 6, "mean_probability": 0.35333333333333333, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 11, "mean_probability": 0.4536363636363637, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 18, "mean_probability": 0.5522222222222222, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 9, "mean_probability": 0.6388888888888888, "observed_success_rate": 0.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.72, "observed_success_rate": 0.0 } ] }, "success_at_decision": { "n": 100, "correct": 55, "accuracy": 0.55, "balanced_accuracy": null, "success_recall": 0.55, "failure_recall": null, "auroc": null, "brier": 0.2387339999999999, "ece_10_equal_width_bins": 0.4736, "confusion_matrix": { "true_success_pred_success": 55, "true_success_pred_fail": 45, "true_fail_pred_success": 0, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.27, "observed_success_rate": 1.0 }, { "lower": 0.3, "upper": 0.4, "count": 17, "mean_probability": 0.3605882352941176, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 27, "mean_probability": 0.43555555555555553, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 12, "mean_probability": 0.5675, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 40, "mean_probability": 0.6380000000000001, "observed_success_rate": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 3, "mean_probability": 0.7166666666666667, "observed_success_rate": 1.0 } ] }, "success_early": { "n": 50, "correct": 31, "accuracy": 0.62, "balanced_accuracy": null, "success_recall": 0.62, "failure_recall": null, "auroc": null, "brier": 0.24056999999999998, "ece_10_equal_width_bins": 0.48219999999999996, "confusion_matrix": { "true_success_pred_success": 31, "true_success_pred_fail": 19, "true_fail_pred_success": 0, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 6, "mean_probability": 0.36166666666666664, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 13, "mean_probability": 0.4476923076923077, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 22, "mean_probability": 0.5522727272727272, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 8, "mean_probability": 0.6287499999999999, "observed_success_rate": 1.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.72, "observed_success_rate": 1.0 } ] } }, "per_task": { "clip_the_socks_to_the_hanger": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.2309, "ece_10_equal_width_bins": 0.47, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.57, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.63, "observed_success_rate": 1.0 } ] }, "fold_a_paper_plane_with_a_square_sheet_of_paper_pick_it_up_from_the_middle": { "n": 3, "correct": 1, "accuracy": 0.3333333333333333, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.36323333333333335, "ece_10_equal_width_bins": 0.3566666666666667, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.675, "observed_success_rate": 0.5 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.72, "observed_success_rate": 0.0 } ] }, "fold_and_stack_the_t_shirts": { "n": 16, "correct": 7, "accuracy": 0.4375, "balanced_accuracy": 0.4375, "success_recall": 0.625, "failure_recall": 0.25, "auroc": 0.515625, "brier": 0.27388125, "ece_10_equal_width_bins": 0.20937500000000003, "confusion_matrix": { "true_success_pred_success": 5, "true_success_pred_fail": 3, "true_fail_pred_success": 6, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.28, "observed_success_rate": 0.0 }, { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.355, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.42000000000000004, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 5, "mean_probability": 0.5720000000000001, "observed_success_rate": 0.4 }, { "lower": 0.6, "upper": 0.7, "count": 6, "mean_probability": 0.6266666666666667, "observed_success_rate": 0.5 } ] }, "fold_and_stack_the_towels": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.27290000000000003, "ece_10_equal_width_bins": 0.050000000000000044, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.44999999999999996, "observed_success_rate": 0.5 } ] }, "fold_and_stack_the_trousers": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.625, "brier": 0.27237500000000003, "ece_10_equal_width_bins": 0.15749999999999997, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 4, "mean_probability": 0.6575, "observed_success_rate": 0.5 } ] }, "fold_the_napkin_into_a_case_and_place_the_utensils_inside": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.28925, "ece_10_equal_width_bins": 0.15500000000000003, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.655, "observed_success_rate": 0.5 } ] }, "fold_the_napkin_place_the_utensils_inside_and_roll_it_up": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.2501, "ece_10_equal_width_bins": 0.49, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.59, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 1.0 } ] }, "insert_the_pens_into_the_pen_caps": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.2405, "ece_10_equal_width_bins": 0.14, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.64, "observed_success_rate": 0.5 } ] }, "insert_the_plug": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.19165, "ece_10_equal_width_bins": 0.425, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.53, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.68, "observed_success_rate": 1.0 } ] }, "insert_the_wireless_bluetooth_earbuds_into_the_charging_case": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.125, "brier": 0.270475, "ece_10_equal_width_bins": 0.2175, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 3, "mean_probability": 0.58, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 0.0 } ] }, "load_the_bowls_into_the_dish_rack": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.29244999999999993, "ece_10_equal_width_bins": 0.16500000000000004, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.33499999999999996, "observed_success_rate": 0.5 } ] }, "load_the_mixed_dishes_into_the_dish_rack": { "n": 1, "correct": 0, "accuracy": 0.0, "balanced_accuracy": null, "success_recall": 0.0, "failure_recall": null, "auroc": null, "brier": 0.31360000000000005, "ece_10_equal_width_bins": 0.56, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.44, "observed_success_rate": 1.0 } ] }, "load_the_plates_into_the_dish_rack": { "n": 10, "correct": 7, "accuracy": 0.7, "balanced_accuracy": 0.7083333333333333, "success_recall": 0.75, "failure_recall": 0.6666666666666666, "auroc": 0.8333333333333334, "brier": 0.21139000000000002, "ece_10_equal_width_bins": 0.267, "confusion_matrix": { "true_success_pred_success": 3, "true_success_pred_fail": 1, "true_fail_pred_success": 2, "true_fail_pred_fail": 4 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.375, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.42, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.58, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 3, "mean_probability": 0.6466666666666666, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.7, "observed_success_rate": 1.0 } ] }, "open_the_zip_top_bag_and_remove_the_fake_food": { "n": 6, "correct": 3, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.6666666666666666, "failure_recall": 0.3333333333333333, "auroc": 0.3333333333333333, "brier": 0.2707, "ece_10_equal_width_bins": 0.22666666666666668, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 1, "true_fail_pred_success": 2, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.38, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.44, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.545, "observed_success_rate": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.605, "observed_success_rate": 0.5 } ] }, "pack_the_chocolate": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.3037, "ece_10_equal_width_bins": 0.54, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.43, "observed_success_rate": 0.0 } ] }, "pack_the_cookies": { "n": 2, "correct": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "success_recall": 1.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.16465000000000002, "ece_10_equal_width_bins": 0.405, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.38, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.57, "observed_success_rate": 1.0 } ] }, "pack_the_luggage": { "n": 7, "correct": 5, "accuracy": 0.7142857142857143, "balanced_accuracy": 0.7083333333333333, "success_recall": 0.6666666666666666, "failure_recall": 0.75, "auroc": 0.75, "brier": 0.21092857142857144, "ece_10_equal_width_bins": 0.09000000000000004, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 3 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.33, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.4266666666666667, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.6, "upper": 0.7, "count": 3, "mean_probability": 0.6733333333333333, "observed_success_rate": 0.6666666666666666 } ] }, "pack_the_student_bag": { "n": 5, "correct": 2, "accuracy": 0.4, "balanced_accuracy": 0.3333333333333333, "success_recall": 0.0, "failure_recall": 0.6666666666666666, "auroc": 0.3333333333333333, "brier": 0.25470000000000004, "ece_10_equal_width_bins": 0.30999999999999994, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 2, "true_fail_pred_success": 1, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.4533333333333333, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.56, "observed_success_rate": 0.0 } ] }, "pack_the_takeout_coffee": { "n": 9, "correct": 4, "accuracy": 0.4444444444444444, "balanced_accuracy": 0.425, "success_recall": 0.6, "failure_recall": 0.25, "auroc": 0.55, "brier": 0.25017777777777783, "ece_10_equal_width_bins": 0.24888888888888883, "confusion_matrix": { "true_success_pred_success": 3, "true_success_pred_fail": 2, "true_fail_pred_success": 3, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.39, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.435, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.5449999999999999, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 3, "mean_probability": 0.6233333333333334, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.72, "observed_success_rate": 1.0 } ] }, "pack_up_the_instrument_practice_props": { "n": 4, "correct": 1, "accuracy": 0.25, "balanced_accuracy": 0.25, "success_recall": 0.5, "failure_recall": 0.0, "auroc": 0.5, "brier": 0.30984999999999996, "ece_10_equal_width_bins": 0.535, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.43, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.645, "observed_success_rate": 0.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.72, "observed_success_rate": 1.0 } ] }, "place_and_organize_the_beverage_onto_the_shelf": { "n": 2, "correct": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "success_recall": 1.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.17285, "ece_10_equal_width_bins": 0.41500000000000004, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.44, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 1.0 } ] }, "place_and_organize_the_canned_foods_onto_the_counter": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.34850000000000003, "ece_10_equal_width_bins": 0.5900000000000001, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.43, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 0.0 } ] }, "place_and_organize_the_chips_bags_onto_the_shelf": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.24370000000000003, "ece_10_equal_width_bins": 0.06000000000000005, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.56, "observed_success_rate": 0.5 } ] }, "place_and_organize_the_fake_fruits_in_the_fruit_bowl": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.2605, "ece_10_equal_width_bins": 0.020000000000000018, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.48, "observed_success_rate": 0.5 } ] }, "place_and_organize_the_pasta_boxes_and_bags_onto_the_shelf": { "n": 2, "correct": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "success_recall": 1.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.15369999999999998, "ece_10_equal_width_bins": 0.39, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.43, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.65, "observed_success_rate": 1.0 } ] }, "place_and_organize_the_shoes_in_the_shoe_cabinet": { "n": 13, "correct": 10, "accuracy": 0.7692307692307693, "balanced_accuracy": 0.7619047619047619, "success_recall": 0.8571428571428571, "failure_recall": 0.6666666666666666, "auroc": 0.9047619047619048, "brier": 0.18079230769230772, "ece_10_equal_width_bins": 0.2561538461538462, "confusion_matrix": { "true_success_pred_success": 6, "true_success_pred_fail": 1, "true_fail_pred_success": 2, "true_fail_pred_fail": 4 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.36, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.42, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.5, "upper": 0.6, "count": 3, "mean_probability": 0.5233333333333333, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.6, "upper": 0.7, "count": 5, "mean_probability": 0.644, "observed_success_rate": 1.0 } ] }, "place_and_organize_the_shoes_onto_the_shoe_shelf": { "n": 6, "correct": 2, "accuracy": 0.3333333333333333, "balanced_accuracy": 0.3333333333333333, "success_recall": 0.3333333333333333, "failure_recall": 0.3333333333333333, "auroc": 0.1111111111111111, "brier": 0.3269333333333333, "ece_10_equal_width_bins": 0.41, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 2, "true_fail_pred_success": 2, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.44999999999999996, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.54, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.625, "observed_success_rate": 0.0 } ] }, "place_the_bread": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.22024999999999997, "ece_10_equal_width_bins": 0.45499999999999996, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.57, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.66, "observed_success_rate": 1.0 } ] }, "place_the_coffee_filter_in_the_dripper": { "n": 10, "correct": 3, "accuracy": 0.3, "balanced_accuracy": 0.30000000000000004, "success_recall": 0.2, "failure_recall": 0.4, "auroc": 0.3, "brier": 0.29973000000000005, "ece_10_equal_width_bins": 0.2869999999999999, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 4, "true_fail_pred_success": 3, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 6, "mean_probability": 0.4283333333333334, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.575, "observed_success_rate": 0.5 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.645, "observed_success_rate": 0.0 } ] }, "place_the_cup_by_the_coaster": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.2605, "ece_10_equal_width_bins": 0.020000000000000018, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.52, "observed_success_rate": 0.5 } ] }, "place_the_flowers_into_the_vase": { "n": 6, "correct": 1, "accuracy": 0.16666666666666666, "balanced_accuracy": 0.16666666666666666, "success_recall": 0.3333333333333333, "failure_recall": 0.0, "auroc": 0.3333333333333333, "brier": 0.29175, "ece_10_equal_width_bins": 0.535, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 2, "true_fail_pred_success": 3, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.37, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.45, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 3, "mean_probability": 0.5466666666666666, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 1.0 } ] }, "place_the_food_in_the_zip_top_bag_and_seal_the_zipper": { "n": 8, "correct": 3, "accuracy": 0.375, "balanced_accuracy": 0.375, "success_recall": 0.25, "failure_recall": 0.5, "auroc": 0.46875, "brier": 0.256275, "ece_10_equal_width_bins": 0.2875, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 3, "true_fail_pred_success": 2, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.37, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 4, "mean_probability": 0.44749999999999995, "observed_success_rate": 0.75 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.52, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.6, "observed_success_rate": 0.5 } ] }, "place_the_letter_into_the_envelope_and_seal_it": { "n": 4, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.36615, "ece_10_equal_width_bins": 0.605, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 2, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.39, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.4, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.59, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.62, "observed_success_rate": 0.0 } ] }, "place_the_mixed_dishes_into_the_plastic_bin": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.28645000000000004, "ece_10_equal_width_bins": 0.535, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.48, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.55, "observed_success_rate": 0.0 } ] }, "place_the_shirt_on_the_hanger": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.5, "brier": 0.2621, "ece_10_equal_width_bins": 0.10999999999999999, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.61, "observed_success_rate": 0.5 } ] }, "place_the_skirt_on_the_hanger": { "n": 1, "correct": 1, "accuracy": 1.0, "balanced_accuracy": null, "success_recall": 1.0, "failure_recall": null, "auroc": null, "brier": 0.1296, "ece_10_equal_width_bins": 0.36, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.64, "observed_success_rate": 1.0 } ] }, "place_the_t_shirt_on_the_hanger": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.25930000000000003, "ece_10_equal_width_bins": 0.16999999999999993, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.6699999999999999, "observed_success_rate": 0.5 } ] }, "place_the_trousers_on_the_hanger": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.25, "brier": 0.2607, "ece_10_equal_width_bins": 0.07499999999999996, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 4, "mean_probability": 0.575, "observed_success_rate": 0.5 } ] }, "put_the_credit_cards_into_the_card_holder": { "n": 7, "correct": 2, "accuracy": 0.2857142857142857, "balanced_accuracy": 0.3333333333333333, "success_recall": 0.0, "failure_recall": 0.6666666666666666, "auroc": 0.20833333333333334, "brier": 0.30604285714285717, "ece_10_equal_width_bins": 0.30142857142857143, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 4, "true_fail_pred_success": 1, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.36, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 4, "mean_probability": 0.435, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.57, "observed_success_rate": 0.0 } ] }, "put_the_phone_into_the_phone_case": { "n": 8, "correct": 2, "accuracy": 0.25, "balanced_accuracy": 0.25, "success_recall": 0.25, "failure_recall": 0.25, "auroc": 0.34375, "brier": 0.3075875, "ece_10_equal_width_bins": 0.35875, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 3, "true_fail_pred_success": 3, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.375, "observed_success_rate": 0.5 }, { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.44999999999999996, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.58, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 3, "mean_probability": 0.6466666666666666, "observed_success_rate": 0.3333333333333333 } ] }, "put_the_photo_into_the_frame": { "n": 2, "correct": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "success_recall": 1.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.1768, "ece_10_equal_width_bins": 0.42000000000000004, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.44, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.6, "observed_success_rate": 1.0 } ] }, "put_the_pillow_into_the_pillowcase": { "n": 4, "correct": 1, "accuracy": 0.25, "balanced_accuracy": 0.25, "success_recall": 0.5, "failure_recall": 0.0, "auroc": 0.5, "brier": 0.3103, "ece_10_equal_width_bins": 0.545, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.59, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.65, "observed_success_rate": 1.0 } ] }, "put_the_plastic_bottles_in_the_bin": { "n": 16, "correct": 8, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.625, "failure_recall": 0.375, "auroc": 0.296875, "brier": 0.29002500000000003, "ece_10_equal_width_bins": 0.33875000000000005, "confusion_matrix": { "true_success_pred_success": 5, "true_success_pred_fail": 3, "true_fail_pred_success": 5, "true_fail_pred_fail": 3 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.375, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 4, "mean_probability": 0.45749999999999996, "observed_success_rate": 0.25 }, { "lower": 0.5, "upper": 0.6, "count": 5, "mean_probability": 0.5559999999999999, "observed_success_rate": 0.8 }, { "lower": 0.6, "upper": 0.7, "count": 5, "mean_probability": 0.624, "observed_success_rate": 0.2 } ] }, "put_the_trash_bags_into_the_trash_bin": { "n": 5, "correct": 3, "accuracy": 0.6, "balanced_accuracy": 0.6666666666666666, "success_recall": 1.0, "failure_recall": 0.3333333333333333, "auroc": 1.0, "brier": 0.22708, "ece_10_equal_width_bins": 0.304, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.46, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 3, "mean_probability": 0.5566666666666666, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.61, "observed_success_rate": 1.0 } ] }, "remove_the_brooch_from_the_garment": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.3506, "ece_10_equal_width_bins": 0.5900000000000001, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.46, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.64, "observed_success_rate": 0.0 } ] }, "remove_the_microphone_from_the_stand": { "n": 4, "correct": 1, "accuracy": 0.25, "balanced_accuracy": 0.25, "success_recall": 0.0, "failure_recall": 0.5, "auroc": 0.0, "brier": 0.32222500000000004, "ece_10_equal_width_bins": 0.5625, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 2, "true_fail_pred_success": 1, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.375, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.44, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.56, "observed_success_rate": 0.0 } ] }, "remove_the_phone_from_the_phone_stand": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.33864999999999995, "ece_10_equal_width_bins": 0.5549999999999999, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.27, "observed_success_rate": 1.0 }, { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.38, "observed_success_rate": 0.0 } ] }, "remove_the_pillowcase_from_the_pillow": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.277, "ece_10_equal_width_bins": 0.13, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.63, "observed_success_rate": 0.5 } ] }, "remove_the_shirt_from_the_hanger": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.32125000000000004, "ece_10_equal_width_bins": 0.5549999999999999, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.56, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.67, "observed_success_rate": 0.0 } ] }, "remove_the_socks_clipped_to_the_hanger": { "n": 20, "correct": 10, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.8, "failure_recall": 0.2, "auroc": 0.485, "brier": 0.25672999999999996, "ece_10_equal_width_bins": 0.124, "confusion_matrix": { "true_success_pred_success": 8, "true_success_pred_fail": 2, "true_fail_pred_success": 8, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.43, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.5, "upper": 0.6, "count": 9, "mean_probability": 0.5577777777777777, "observed_success_rate": 0.5555555555555556 }, { "lower": 0.6, "upper": 0.7, "count": 7, "mean_probability": 0.6285714285714287, "observed_success_rate": 0.42857142857142855 } ] }, "remove_the_underwear_clipped_to_the_hanger": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": null, "success_recall": null, "failure_recall": 0.0, "auroc": null, "brier": 0.37845, "ece_10_equal_width_bins": 0.615, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.615, "observed_success_rate": 0.0 } ] }, "sort_the_eating_utensils_into_containers": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.25, "ece_10_equal_width_bins": 0.14, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.36, "observed_success_rate": 0.5 } ] }, "sort_the_legos_into_containers_by_color": { "n": 10, "correct": 5, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.8, "failure_recall": 0.2, "auroc": 0.7, "brier": 0.25117999999999996, "ece_10_equal_width_bins": 0.32599999999999996, "confusion_matrix": { "true_success_pred_success": 4, "true_success_pred_fail": 1, "true_fail_pred_success": 4, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.33499999999999996, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 5, "mean_probability": 0.5539999999999999, "observed_success_rate": 0.2 }, { "lower": 0.6, "upper": 0.7, "count": 3, "mean_probability": 0.6133333333333333, "observed_success_rate": 1.0 } ] }, "sort_the_stationery_into_containers": { "n": 4, "correct": 1, "accuracy": 0.25, "balanced_accuracy": 0.25, "success_recall": 0.0, "failure_recall": 0.5, "auroc": 0.5, "brier": 0.26455000000000006, "ece_10_equal_width_bins": 0.31, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 2, "true_fail_pred_success": 1, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.43, "observed_success_rate": 0.6666666666666666 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.53, "observed_success_rate": 0.0 } ] }, "sort_the_tools_into_containers": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.2627999999999999, "ece_10_equal_width_bins": 0.18, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.32, "observed_success_rate": 0.5 } ] }, "stack_the_books_and_files": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.5, "brier": 0.254725, "ece_10_equal_width_bins": 0.48250000000000004, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.58, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.62, "observed_success_rate": 0.0 }, { "lower": 0.7, "upper": 0.8, "count": 1, "mean_probability": 0.73, "observed_success_rate": 1.0 } ] }, "stack_the_hats": { "n": 7, "correct": 6, "accuracy": 0.8571428571428571, "balanced_accuracy": 0.875, "success_recall": 0.75, "failure_recall": 1.0, "auroc": 0.9166666666666666, "brier": 0.18265714285714285, "ece_10_equal_width_bins": 0.2542857142857143, "confusion_matrix": { "true_success_pred_success": 3, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 3 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.42, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.54, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 2, "mean_probability": 0.645, "observed_success_rate": 1.0 } ] }, "take_the_beverage_out_of_the_canvas_bag": { "n": 4, "correct": 3, "accuracy": 0.75, "balanced_accuracy": 0.75, "success_recall": 0.5, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.17030000000000003, "ece_10_equal_width_bins": 0.405, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 2, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.46, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.62, "observed_success_rate": 1.0 } ] }, "take_the_credit_cards_out_of_the_card_holder": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.2768, "ece_10_equal_width_bins": 0.07999999999999996, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.42000000000000004, "observed_success_rate": 0.5 } ] }, "take_the_fake_fruits_out_of_the_plastic_bag": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.0, "failure_recall": 1.0, "auroc": 0.0, "brier": 0.29345, "ece_10_equal_width_bins": 0.535, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.38, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.45, "observed_success_rate": 0.0 } ] }, "take_the_phone_out_of_the_phone_case": { "n": 2, "correct": 2, "accuracy": 1.0, "balanced_accuracy": 1.0, "success_recall": 1.0, "failure_recall": 1.0, "auroc": 1.0, "brier": 0.178, "ece_10_equal_width_bins": 0.42000000000000004, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 0, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.46, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.62, "observed_success_rate": 1.0 } ] }, "take_the_snacks_out_of_the_paper_bag": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.37250000000000005, "ece_10_equal_width_bins": 0.6100000000000001, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.41, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.63, "observed_success_rate": 0.0 } ] }, "take_the_wine_glass_off_the_wine_glass_rack": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.5, "failure_recall": 0.5, "auroc": 0.625, "brier": 0.226425, "ece_10_equal_width_bins": 0.25749999999999995, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.48, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.58, "observed_success_rate": 0.5 } ] }, "unload_the_bowls_from_the_dish_rack": { "n": 6, "correct": 5, "accuracy": 0.8333333333333334, "balanced_accuracy": 0.8333333333333333, "success_recall": 0.6666666666666666, "failure_recall": 1.0, "auroc": 0.6666666666666666, "brier": 0.22855000000000003, "ece_10_equal_width_bins": 0.4716666666666667, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 3 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.36, "observed_success_rate": 1.0 }, { "lower": 0.4, "upper": 0.5, "count": 3, "mean_probability": 0.4366666666666667, "observed_success_rate": 0.0 }, { "lower": 0.5, "upper": 0.6, "count": 2, "mean_probability": 0.56, "observed_success_rate": 1.0 } ] }, "unload_the_cups_from_the_dish_rack": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 0.5, "failure_recall": 0.5, "auroc": 0.5, "brier": 0.24824999999999997, "ece_10_equal_width_bins": 0.49, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 1 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.35, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 1, "mean_probability": 0.48, "observed_success_rate": 1.0 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.51, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.6, "observed_success_rate": 0.0 } ] }, "unload_the_mixed_dishes_from_the_dish_rack": { "n": 2, "correct": 0, "accuracy": 0.0, "balanced_accuracy": 0.0, "success_recall": 0.0, "failure_recall": 0.0, "auroc": 0.0, "brier": 0.43599999999999994, "ece_10_equal_width_bins": 0.6599999999999999, "confusion_matrix": { "true_success_pred_success": 0, "true_success_pred_fail": 1, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.32, "observed_success_rate": 1.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.64, "observed_success_rate": 0.0 } ] }, "unload_the_plates_from_the_dish_rack": { "n": 2, "correct": 1, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 1.0, "brier": 0.18964999999999999, "ece_10_equal_width_bins": 0.425, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 0, "true_fail_pred_success": 1, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.52, "observed_success_rate": 0.0 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.67, "observed_success_rate": 1.0 } ] }, "unpack_the_express_box": { "n": 4, "correct": 3, "accuracy": 0.75, "balanced_accuracy": 0.75, "success_recall": 0.5, "failure_recall": 1.0, "auroc": 0.75, "brier": 0.21915, "ece_10_equal_width_bins": 0.225, "confusion_matrix": { "true_success_pred_success": 1, "true_success_pred_fail": 1, "true_fail_pred_success": 0, "true_fail_pred_fail": 2 }, "calibration": [ { "lower": 0.3, "upper": 0.4, "count": 1, "mean_probability": 0.39, "observed_success_rate": 0.0 }, { "lower": 0.4, "upper": 0.5, "count": 2, "mean_probability": 0.47, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 1, "mean_probability": 0.55, "observed_success_rate": 1.0 } ] }, "unzip_the_laundry_bag_and_remove_the_bra": { "n": 4, "correct": 2, "accuracy": 0.5, "balanced_accuracy": 0.5, "success_recall": 1.0, "failure_recall": 0.0, "auroc": 0.75, "brier": 0.245875, "ece_10_equal_width_bins": 0.27249999999999996, "confusion_matrix": { "true_success_pred_success": 2, "true_success_pred_fail": 0, "true_fail_pred_success": 2, "true_fail_pred_fail": 0 }, "calibration": [ { "lower": 0.5, "upper": 0.6, "count": 3, "mean_probability": 0.5733333333333333, "observed_success_rate": 0.3333333333333333 }, { "lower": 0.6, "upper": 0.7, "count": 1, "mean_probability": 0.63, "observed_success_rate": 1.0 } ] } }, "per_difficulty": { "at_decision": { "n": 206, "correct": 97, "accuracy": 0.470873786407767, "balanced_accuracy": 0.47311320754716985, "success_recall": 0.55, "failure_recall": 0.39622641509433965, "auroc": 0.5221698113207547, "brier": 0.2630766990291263, "ece_10_equal_width_bins": 0.14281553398058267, "confusion_matrix": { "true_success_pred_success": 55, "true_success_pred_fail": 45, "true_fail_pred_success": 64, "true_fail_pred_fail": 42 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.27, "observed_success_rate": 1.0 }, { "lower": 0.3, "upper": 0.4, "count": 32, "mean_probability": 0.35937499999999994, "observed_success_rate": 0.53125 }, { "lower": 0.4, "upper": 0.5, "count": 54, "mean_probability": 0.43500000000000005, "observed_success_rate": 0.5 }, { "lower": 0.5, "upper": 0.6, "count": 40, "mean_probability": 0.565, "observed_success_rate": 0.3 }, { "lower": 0.6, "upper": 0.7, "count": 76, "mean_probability": 0.6346052631578951, "observed_success_rate": 0.5263157894736842 }, { "lower": 0.7, "upper": 0.8, "count": 3, "mean_probability": 0.7166666666666667, "observed_success_rate": 1.0 } ] }, "early": { "n": 96, "correct": 49, "accuracy": 0.5104166666666666, "balanced_accuracy": 0.5056521739130435, "success_recall": 0.62, "failure_recall": 0.391304347826087, "auroc": 0.49369565217391304, "brier": 0.2585739583333332, "ece_10_equal_width_bins": 0.07802083333333333, "confusion_matrix": { "true_success_pred_success": 31, "true_success_pred_fail": 19, "true_fail_pred_success": 28, "true_fail_pred_fail": 18 }, "calibration": [ { "lower": 0.2, "upper": 0.3, "count": 1, "mean_probability": 0.28, "observed_success_rate": 0.0 }, { "lower": 0.3, "upper": 0.4, "count": 12, "mean_probability": 0.3575, "observed_success_rate": 0.5 }, { "lower": 0.4, "upper": 0.5, "count": 24, "mean_probability": 0.4504166666666667, "observed_success_rate": 0.5416666666666666 }, { "lower": 0.5, "upper": 0.6, "count": 40, "mean_probability": 0.5522500000000001, "observed_success_rate": 0.55 }, { "lower": 0.6, "upper": 0.7, "count": 17, "mean_probability": 0.6341176470588235, "observed_success_rate": 0.47058823529411764 }, { "lower": 0.7, "upper": 0.8, "count": 2, "mean_probability": 0.72, "observed_success_rate": 0.5 } ] } } }