{ "meta": { "generated_at_utc": "2026-02-24T16:23:15.081119+00:00", "manifest": "family_holdout_protocol_v1", "protocol": "S2_family_holdout", "validation_data": "materials_validation_dataset_v1", "adapters": [ "flux", "aflow", "jarvis", "matbench" ], "resolved_adapters": [ "flux", "aflow", "jarvis", "matbench" ], "selected_fold_ids": "all", "num_folds": 19, "holdout_mode": "strict" }, "fold_results": [ { "fold_id": "family:alkali_halides", "holdout_key": "alkali_halides", "holdout_size": 20, "train_size": 160, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.11683807959723662, "MAPE": 0.5304746875935853, "median_APE": 0.453440827489514, "p90_AE": 0.25343203767567957 }, "theta_D_K": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 6.862521824479484, "MAPE": 2.633732940543207, "median_APE": 2.1257327477783514, "p90_AE": 9.875266516182535 }, "v_sound_m_s": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 26.239877625603448, "MAPE": 0.7766905596985743, "median_APE": 0.6612282715929227, "p90_AE": 48.15274290924981 }, "density_g_cm3": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.03464343374377321, "MAPE": 1.079036940442693, "median_APE": 0.9149620625216637, "p90_AE": 0.0733606463534926 }, "kappa_W_mK": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.025066694917451737, "MAPE": 0.7248512324300987, "median_APE": 0.39790522499378256, "p90_AE": 0.04100065813848275 } }, "aggregate": { "N_points": 100, "overall_MAPE": 1.1489572721416317, "overall_median_APE": 0.6732846033263149 }, "runtime": { "N_calls": 20, "mean_ms": 0.4488049977226183, "p95_ms": 0.47889997949823737 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 1.2365615000000008, "MAPE": 6.595181405980109, "median_APE": 3.359387019230769, "p90_AE": 2.955599999999997 }, "theta_D_K": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 27.006262500000002, "MAPE": 10.528922984732905, "median_APE": 9.457163800099474, "p90_AE": 50.70100000000002 }, "v_sound_m_s": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 1173.1345000000003, "MAPE": 33.54591391813985, "median_APE": 36.90666809360731, "p90_AE": 1805.92 }, "density_g_cm3": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.209082, "MAPE": 5.7258098274199565, "median_APE": 3.711823593073598, "p90_AE": 0.5966900000000006 }, "kappa_W_mK": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 3.1315616250000007, "MAPE": 61.238079808690586, "median_APE": 70.70463675213676, "p90_AE": 5.858804999999999 } }, "aggregate": { "N_points": 100, "overall_MAPE": 23.526781588992677, "overall_median_APE": 11.333415194518413 }, "runtime": { "N_calls": 20, "mean_ms": 0.002780000795610249, "p95_ms": 0.00300002284348011 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 16, "coverage_pct": 80.0, "MAE": 4.4340625000000005, "MAPE": 19.090029883590873, "median_APE": 21.6515873015873, "p90_AE": 8.780000000000001 }, "theta_D_K": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 20, "N_pred": 16, "coverage_pct": 80.0, "MAE": 0.13775, "MAPE": 4.768199410730345, "median_APE": 2.4261426978818323, "p90_AE": 0.27600000000000025 }, "kappa_W_mK": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 32, "overall_MAPE": 11.929114647160606, "overall_median_APE": 9.437500000000004 }, "runtime": { "N_calls": 20, "mean_ms": 0.0018900143913924694, "p95_ms": 0.002500019036233425 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 5.575, "MAPE": 34.67532718415954, "median_APE": 22.1774193548387, "p90_AE": 10.0 }, "theta_D_K": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 20, "overall_MAPE": 34.67532718415954, "overall_median_APE": 22.1774193548387 }, "runtime": { "N_calls": 20, "mean_ms": 0.0022050022380426526, "p95_ms": 0.002400018274784088 } } ] }, { "fold_id": "family:alkaline_earth_chalcogenides", "holdout_key": "alkaline_earth_chalcogenides", "holdout_size": 20, "train_size": 160, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.3205058901235388, "MAPE": 0.42378830170823195, "median_APE": 0.31968200382506906, "p90_AE": 0.6158795352756954 }, "theta_D_K": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 6.799290767177124, "MAPE": 1.3009330945596147, "median_APE": 0.5720304380027326, "p90_AE": 17.06973290001997 }, "v_sound_m_s": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 90.08312162089061, "MAPE": 1.6532555389571595, "median_APE": 1.1377531374156737, "p90_AE": 159.64982628641155 }, "density_g_cm3": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.025321660480428367, "MAPE": 0.5682796686655969, "median_APE": 0.37226794964137555, "p90_AE": 0.0677179882481278 }, "kappa_W_mK": { "N_truth": 5, "N_pred": 5, "coverage_pct": 100.0, "MAE": 0.07394439499932207, "MAPE": 0.13505345292839174, "median_APE": 0.0640881245598616, "p90_AE": 0.22216417024657176 } }, "aggregate": { "N_points": 85, "overall_MAPE": 0.9364752863818119, "overall_median_APE": 0.5073275534509212 }, "runtime": { "N_calls": 20, "mean_ms": 0.31971000717021525, "p95_ms": 0.36810000892728567 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 7.3155375000000005, "MAPE": 8.821640138730531, "median_APE": 7.017159340659338, "p90_AE": 12.422249999999991 }, "theta_D_K": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 69.70962499999999, "MAPE": 14.403724568648453, "median_APE": 11.579067647058825, "p90_AE": 209.572 }, "v_sound_m_s": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 1163.76425, "MAPE": 20.728582777973536, "median_APE": 19.36075591615956, "p90_AE": 2560.49 }, "density_g_cm3": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.2675915000000001, "MAPE": 6.757167150250669, "median_APE": 3.739887535235745, "p90_AE": 0.66832 }, "kappa_W_mK": { "N_truth": 5, "N_pred": 5, "coverage_pct": 100.0, "MAE": 74.90426, "MAPE": 54.68195045045045, "median_APE": 77.29375, "p90_AE": 316.7715 } }, "aggregate": { "N_points": 85, "overall_MAPE": 15.148612293697836, "overall_median_APE": 8.844933333333339 }, "runtime": { "N_calls": 20, "mean_ms": 0.003110003308393061, "p95_ms": 0.005500041879713535 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 7.825750000000001, "MAPE": 11.418105149986394, "median_APE": 10.671428571428567, "p90_AE": 18.709999999999994 }, "theta_D_K": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 0.29545000000000005, "MAPE": 6.955374658831562, "median_APE": 3.820003827484828, "p90_AE": 0.8005000000000004 }, "kappa_W_mK": { "N_truth": 5, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 40, "overall_MAPE": 9.186739904408977, "overall_median_APE": 8.005042016806717 }, "runtime": { "N_calls": 20, "mean_ms": 0.0024449953343719244, "p95_ms": 0.002900022082030773 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 20, "N_pred": 20, "coverage_pct": 100.0, "MAE": 13.075000000000003, "MAPE": 14.227025660052107, "median_APE": 11.75675675675675, "p90_AE": 43.00000000000001 }, "theta_D_K": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 20, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 5, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 20, "overall_MAPE": 14.227025660052107, "overall_median_APE": 11.75675675675675 }, "runtime": { "N_calls": 20, "mean_ms": 0.002410015440545976, "p95_ms": 0.002600019797682762 } } ] }, { "fold_id": "family:alkaline_earth_halides", "holdout_key": "alkaline_earth_halides", "holdout_size": 4, "train_size": 176, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.058620528951491835, "MAPE": 0.07086709574117808, "median_APE": 0.06956971805781448, "p90_AE": 0.13474776944477185 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 3.7791731835689717, "MAPE": 0.9209179815261884, "median_APE": 0.9587339958185626, "p90_AE": 4.606126488759969 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 19.49242910862472, "MAPE": 0.3667313468608408, "median_APE": 0.25987926349257806, "p90_AE": 46.641171887393284 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.0136576156123146, "MAPE": 0.3227239703801391, "median_APE": 0.18275660254483939, "p90_AE": 0.03736439073083009 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.02517168267759784, "MAPE": 0.24654380958395822, "median_APE": 0.2814412816568283, "p90_AE": 0.03439509701344434 } }, "aggregate": { "N_points": 20, "overall_MAPE": 0.3855568408184609, "overall_median_APE": 0.26207211235218963 }, "runtime": { "N_calls": 4, "mean_ms": 0.27372500335332006, "p95_ms": 0.298400002066046 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 3.5780000000000047, "MAPE": 4.5671229758966145, "median_APE": 3.724214340198328, "p90_AE": 6.094800000000006 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 45.01774999999999, "MAPE": 10.147544767383348, "median_APE": 10.580419644493308, "p90_AE": 61.19799999999998 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 1157.4625, "MAPE": 22.70306614776218, "median_APE": 24.028921568627446, "p90_AE": 2328.88 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.13496999999999992, "MAPE": 3.5605325413821034, "median_APE": 3.8282584789906173, "p90_AE": 0.1833399999999994 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 5.9429324999999995, "MAPE": 55.97956249999999, "median_APE": 52.604375000000005, "p90_AE": 9.1674 } }, "aggregate": { "N_points": 20, "overall_MAPE": 19.39156578648485, "overall_median_APE": 9.189289215686275 }, "runtime": { "N_calls": 4, "mean_ms": 0.003950015525333583, "p95_ms": 0.006099988240748644 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 2.585000000000001, "MAPE": 3.8651636247962142, "median_APE": 3.9471947194719488, "p90_AE": 4.829999999999998 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.17637499999999995, "MAPE": 4.880116091754447, "median_APE": 3.470924007408259, "p90_AE": 0.355 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 4.37263985827533, "overall_median_APE": 3.470924007408259 }, "runtime": { "N_calls": 4, "mean_ms": 0.002900007530115545, "p95_ms": 0.003500026650726795 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 4.7499999999999964, "MAPE": 6.0574422665171825, "median_APE": 5.8688477543406545, "p90_AE": 6.0 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 4, "overall_MAPE": 6.0574422665171825, "overall_median_APE": 5.8688477543406545 }, "runtime": { "N_calls": 4, "mean_ms": 0.002674991264939308, "p95_ms": 0.002700020559132099 } } ] }, { "fold_id": "family:bcc_metals", "holdout_key": "bcc_metals", "holdout_size": 7, "train_size": 173, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.3098103202895044, "MAPE": 0.6716404906072262, "median_APE": 0.5222115285443252, "p90_AE": 2.720920598206476 }, "theta_D_K": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 4.753051983021343, "MAPE": 1.2419890082960383, "median_APE": 1.056729231569433, "p90_AE": 9.286531579885263 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 42.736552864927816, "MAPE": 0.7791830384867755, "median_APE": 0.4617261007865163, "p90_AE": 104.17821650796668 }, "density_g_cm3": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.24166606715862812, "MAPE": 2.049517561960149, "median_APE": 0.9237520260654802, "p90_AE": 1.0019922353145727 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.39796175460735334, "MAPE": 0.40287793071375144, "median_APE": 0.29260614174900645, "p90_AE": 0.8836366679955745 } }, "aggregate": { "N_points": 35, "overall_MAPE": 1.0290416060127878, "overall_median_APE": 0.5221170436996303 }, "runtime": { "N_calls": 7, "mean_ms": 0.4572571571251111, "p95_ms": 0.4879000480286777 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 3, "coverage_pct": 42.857142857142854, "MAE": 44.6543, "MAPE": 26.12399243033226, "median_APE": 4.577500000000001, "p90_AE": 110.93289999999999 }, "theta_D_K": { "N_truth": 7, "N_pred": 3, "coverage_pct": 42.857142857142854, "MAE": 136.65800000000002, "MAPE": 23.58191666666667, "median_APE": 12.940750000000008, "p90_AE": 354.543 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 3, "coverage_pct": 42.857142857142854, "MAE": 2857.5633333333335, "MAPE": 50.942480428548976, "median_APE": 46.52390243902439, "p90_AE": 4426.610000000001 }, "density_g_cm3": { "N_truth": 7, "N_pred": 3, "coverage_pct": 42.857142857142854, "MAE": 1.6411933333333337, "MAPE": 17.75408443141406, "median_APE": 6.1309309309309175, "p90_AE": 3.082280000000001 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 3, "coverage_pct": 42.857142857142854, "MAE": 94.00245, "MAPE": 90.35970456760316, "median_APE": 95.247649122807, "p90_AE": 136.72390000000001 } }, "aggregate": { "N_points": 15, "overall_MAPE": 41.75243570491303, "overall_median_APE": 42.88505747126437 }, "runtime": { "N_calls": 7, "mean_ms": 0.0021000076750559466, "p95_ms": 0.005399982910603285 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 28.965714285714284, "MAPE": 16.687787266899445, "median_APE": 13.20061728395061, "p90_AE": 102.32999999999998 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.3207857142857141, "MAPE": 3.10138657887741, "median_APE": 2.7788649706457993, "p90_AE": 0.49849999999999994 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 14, "overall_MAPE": 9.894586922888427, "overall_median_APE": 3.213738073992724 }, "runtime": { "N_calls": 7, "mean_ms": 0.0027571555360087325, "p95_ms": 0.003400025889277458 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 32.85714285714287, "MAPE": 18.768139170663794, "median_APE": 10.493827160493863, "p90_AE": 94.5 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 7, "overall_MAPE": 18.768139170663794, "overall_median_APE": 10.493827160493863 }, "runtime": { "N_calls": 7, "mean_ms": 0.0025857090284781797, "p95_ms": 0.0028999638743698597 } } ] }, { "fold_id": "family:category_bcc_metal", "holdout_key": "category_bcc_metal", "holdout_size": 7, "train_size": 173, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 2.0689052809927877, "MAPE": 2.612711242813167, "median_APE": 0.8524675666156094, "p90_AE": 13.642273494619957 }, "theta_D_K": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 3.8714533184309943, "MAPE": 2.415214418856622, "median_APE": 1.967312081529957, "p90_AE": 15.389392431271233 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 13.600154398185184, "MAPE": 0.47354217101218354, "median_APE": 0.30345676357286777, "p90_AE": 32.437192982712986 }, "density_g_cm3": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.062387732812682124, "MAPE": 3.521161479314636, "median_APE": 0.7620925889390443, "p90_AE": 0.20849309004446526 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.2175863947453587, "MAPE": 1.7203992565308324, "median_APE": 0.18211430298415815, "p90_AE": 7.564111785245572 } }, "aggregate": { "N_points": 35, "overall_MAPE": 2.1486057137054884, "overall_median_APE": 0.6728161312354226 }, "runtime": { "N_calls": 7, "mean_ms": 0.2585285692475736, "p95_ms": 0.28470001416280866 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.4865214285714283, "MAPE": 24.224793453030408, "median_APE": 25.999999999999996, "p90_AE": 2.9884000000000004 }, "theta_D_K": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 26.767264285714294, "MAPE": 12.067625448535612, "median_APE": 14.096518987341758, "p90_AE": 69.572 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1136.096, "MAPE": 37.458475398168446, "median_APE": 36.873666666666665, "p90_AE": 2212.42 }, "density_g_cm3": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.06664128571428575, "MAPE": 4.381078363297844, "median_APE": 2.595953002610957, "p90_AE": 0.14613999999999994 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 69.38357514285714, "MAPE": 93.60610050609453, "median_APE": 99.305609929078, "p90_AE": 140.02091 } }, "aggregate": { "N_points": 35, "overall_MAPE": 34.34761463382537, "overall_median_APE": 25.999999999999996 }, "runtime": { "N_calls": 7, "mean_ms": 0.003414278450821127, "p95_ms": 0.005399982910603285 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 6, "coverage_pct": 85.71428571428571, "MAE": 2.4966666666666644, "MAPE": 18.13975147611345, "median_APE": 23.169841269841264, "p90_AE": 8.669999999999987 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.1287142857142858, "MAPE": 7.7125070032015355, "median_APE": 8.159268929503916, "p90_AE": 0.33000000000000007 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 13, "overall_MAPE": 12.525081375314727, "overall_median_APE": 9.3632958801498 }, "runtime": { "N_calls": 7, "mean_ms": 0.0028142905128853662, "p95_ms": 0.003400025889277458 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.8000000000000078, "MAPE": 22.57572858313378, "median_APE": 24.999999999999993, "p90_AE": 4.000000000000057 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 7, "overall_MAPE": 22.57572858313378, "overall_median_APE": 24.999999999999993 }, "runtime": { "N_calls": 7, "mean_ms": 0.002671432282243456, "p95_ms": 0.0031999661587178707 } } ] }, { "fold_id": "family:category_binary_halide", "holdout_key": "category_binary_halide", "holdout_size": 9, "train_size": 171, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 0.3203053946349262, "MAPE": 0.5025630124718834, "median_APE": 0.34704174252289693, "p90_AE": 1.0793723392249888 }, "theta_D_K": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 1.6326817045089936, "MAPE": 0.5718847655272407, "median_APE": 0.46490646106458433, "p90_AE": 5.343518254647108 }, "v_sound_m_s": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 21.305227369833826, "MAPE": 0.6443531168257348, "median_APE": 0.653277445917035, "p90_AE": 51.572016229831206 }, "density_g_cm3": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 0.024369910578937965, "MAPE": 0.5054152153487502, "median_APE": 0.6235715027064994, "p90_AE": 0.039186459365780024 }, "kappa_W_mK": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 0.004668082773847732, "MAPE": 0.30080309548593587, "median_APE": 0.2102552030137872, "p90_AE": 0.011396352500807616 } }, "aggregate": { "N_points": 45, "overall_MAPE": 0.5050038411319091, "overall_median_APE": 0.3759155057811995 }, "runtime": { "N_calls": 9, "mean_ms": 0.3756444332086378, "p95_ms": 0.41179999243468046 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 9, "N_pred": 6, "coverage_pct": 66.66666666666667, "MAE": 5.9238333333333335, "MAPE": 9.479985073115209, "median_APE": 5.489897727272731, "p90_AE": 22.17349999999999 }, "theta_D_K": { "N_truth": 9, "N_pred": 6, "coverage_pct": 66.66666666666667, "MAE": 21.412000000000006, "MAPE": 11.805464513400992, "median_APE": 13.305292349726777, "p90_AE": 28.462999999999994 }, "v_sound_m_s": { "N_truth": 9, "N_pred": 6, "coverage_pct": 66.66666666666667, "MAE": 543.4575, "MAPE": 19.224762674265587, "median_APE": 22.82363225010284, "p90_AE": 1331.48 }, "density_g_cm3": { "N_truth": 9, "N_pred": 6, "coverage_pct": 66.66666666666667, "MAE": 0.24343333333333325, "MAPE": 5.081488144622537, "median_APE": 3.088101558938318, "p90_AE": 0.6369099999999994 }, "kappa_W_mK": { "N_truth": 9, "N_pred": 6, "coverage_pct": 66.66666666666667, "MAE": 0.5502180833333333, "MAPE": 50.305510416666664, "median_APE": 47.43793333333333, "p90_AE": 1.0222099999999998 } }, "aggregate": { "N_points": 30, "overall_MAPE": 19.179442164414198, "overall_median_APE": 14.292250000000006 }, "runtime": { "N_calls": 9, "mean_ms": 0.0024111230029828018, "p95_ms": 0.005099980626255274 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 21.613333333333333, "MAPE": 31.08732235055608, "median_APE": 31.437500000000007, "p90_AE": 57.400000000000006 }, "theta_D_K": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 0.2881666666666668, "MAPE": 6.467590564684094, "median_APE": 4.967741935483868, "p90_AE": 0.6580000000000004 }, "kappa_W_mK": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 18, "overall_MAPE": 18.777456457620087, "overall_median_APE": 13.290632381583862 }, "runtime": { "N_calls": 9, "mean_ms": 0.0027666553958422607, "p95_ms": 0.0031999661587178707 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 9, "N_pred": 9, "coverage_pct": 100.0, "MAE": 9.22222222222222, "MAPE": 14.964729696569155, "median_APE": 11.11111111111111, "p90_AE": 27.5 }, "theta_D_K": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 9, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 9, "overall_MAPE": 14.964729696569155, "overall_median_APE": 11.11111111111111 }, "runtime": { "N_calls": 9, "mean_ms": 0.0024666660465300083, "p95_ms": 0.002600019797682762 } } ] }, { "fold_id": "family:category_elemental_semiconductor", "holdout_key": "category_elemental_semiconductor", "holdout_size": 4, "train_size": 176, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.5932931482088701, "MAPE": 0.48024760173269304, "median_APE": 0.5121350055603939, "p90_AE": 1.020510199762839 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 31.214842963035487, "MAPE": 3.587558275247859, "median_APE": 4.392350220621693, "p90_AE": 95.69964960112975 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 120.76635106955712, "MAPE": 1.7818193786103889, "median_APE": 1.4096483410477225, "p90_AE": 354.0314930784625 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.082152128678458, "MAPE": 2.270472310525028, "median_APE": 0.5098692139039516, "p90_AE": 0.28204163452583675 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.6238702631128739, "MAPE": 0.15745696943685467, "median_APE": 0.10411163939392357, "p90_AE": 1.7884827613893322 } }, "aggregate": { "N_points": 20, "overall_MAPE": 1.6555109071105645, "overall_median_APE": 0.5118559814928094 }, "runtime": { "N_calls": 4, "mean_ms": 0.25955001183319837, "p95_ms": 0.2707000239752233 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 22.340425000000003, "MAPE": 18.967186648806017, "median_APE": 17.60153197278911, "p90_AE": 40.73000000000002 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 70.40287499999995, "MAPE": 9.518778958336002, "median_APE": 8.398908422459892, "p90_AE": 135.13999999999987 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 2790.54875, "MAPE": 31.889602993379896, "median_APE": 33.975092592592596, "p90_AE": 5233.700000000001 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.17545125000000017, "MAPE": 4.6588899377693735, "median_APE": 4.4805677554315615, "p90_AE": 0.22495000000000065 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 583.09762, "MAPE": 93.7305154590683, "median_APE": 93.97797818181819, "p90_AE": 2072.909 } }, "aggregate": { "N_points": 20, "overall_MAPE": 31.752994799471924, "overall_median_APE": 17.60153197278911 }, "runtime": { "N_calls": 4, "mean_ms": 0.00367498432751745, "p95_ms": 0.004699977580457926 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 51.535000000000004, "MAPE": 23.71956381992355, "median_APE": 20.787356321839084, "p90_AE": 162.0 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.796625, "MAPE": 18.481866988240412, "median_APE": 15.851696196910108, "p90_AE": 1.2510000000000003 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 21.100715404081978, "overall_median_APE": 20.571296240960983 }, "runtime": { "N_calls": 4, "mean_ms": 0.002949993358924985, "p95_ms": 0.0035999692045152187 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 93.0, "MAPE": 33.928237567308386, "median_APE": 23.011494252873565, "p90_AE": 325.0 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 4, "overall_MAPE": 33.928237567308386, "overall_median_APE": 23.011494252873565 }, "runtime": { "N_calls": 4, "mean_ms": 0.0026500056264922023, "p95_ms": 0.002800021320581436 } } ] }, { "fold_id": "family:category_fcc_metal", "holdout_key": "category_fcc_metal", "holdout_size": 4, "train_size": 176, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.6222538811234459, "MAPE": 2.3729167946830136, "median_APE": 1.653236820459962, "p90_AE": 1.4676279684402118 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 3.557882656122974, "MAPE": 2.5550359165366214, "median_APE": 2.213029688481983, "p90_AE": 5.9431198711895235 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 71.46496839839278, "MAPE": 1.2019602627315313, "median_APE": 0.33713254742948373, "p90_AE": 262.14243408191123 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.05047014701692887, "MAPE": 1.3628965696671904, "median_APE": 0.7148192953450346, "p90_AE": 0.10761995009200875 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 6.901520346668477, "MAPE": 4.196605136690811, "median_APE": 3.001200177900663, "p90_AE": 25.209957798874115 } }, "aggregate": { "N_points": 20, "overall_MAPE": 2.337882936061834, "overall_median_APE": 1.0809032777788572 }, "runtime": { "N_calls": 4, "mean_ms": 0.22285000886768103, "p95_ms": 0.25350000942125916 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 2, "coverage_pct": 50.0, "MAE": 0.49819999999999975, "MAPE": 3.6750735294117636, "median_APE": 3.6750735294117636, "p90_AE": 0.6074999999999999 }, "theta_D_K": { "N_truth": 4, "N_pred": 2, "coverage_pct": 50.0, "MAE": 24.301999999999992, "MAPE": 13.789486838213541, "median_APE": 13.789486838213541, "p90_AE": 26.260999999999996 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 2, "coverage_pct": 50.0, "MAE": 2115.9325, "MAPE": 52.167117021276596, "median_APE": 52.167117021276596, "p90_AE": 2272.32 }, "density_g_cm3": { "N_truth": 4, "N_pred": 2, "coverage_pct": 50.0, "MAE": 0.02150249999999998, "MAPE": 1.1927388807429118, "median_APE": 1.1927388807429118, "p90_AE": 0.02839999999999998 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 2, "coverage_pct": 50.0, "MAE": 111.8746525, "MAPE": 93.63967181947406, "median_APE": 93.63967181947406, "p90_AE": 191.55728 } }, "aggregate": { "N_points": 10, "overall_MAPE": 32.89281761782378, "overall_median_APE": 13.789486838213541 }, "runtime": { "N_calls": 4, "mean_ms": 0.002325003151781857, "p95_ms": 0.004500034265220165 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 2.5687499999999965, "MAPE": 4.656876766724988, "median_APE": 4.596362229102158, "p90_AE": 6.069999999999993 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.1879999999999999, "MAPE": 4.7577124935995885, "median_APE": 4.908943505717693, "p90_AE": 0.4109999999999996 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 4.707294630162288, "overall_median_APE": 4.908943505717693 }, "runtime": { "N_calls": 4, "mean_ms": 0.003049994120374322, "p95_ms": 0.0037999707274138927 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 3.625000000000004, "MAPE": 6.378718535469115, "median_APE": 4.605263157894753, "p90_AE": 7.5 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 4, "overall_MAPE": 6.378718535469115, "overall_median_APE": 4.605263157894753 }, "runtime": { "N_calls": 4, "mean_ms": 0.0024999899324029684, "p95_ms": 0.002700020559132099 } } ] }, { "fold_id": "family:category_ionic_crystal", "holdout_key": "category_ionic_crystal", "holdout_size": 10, "train_size": 170, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 1.5178277345988078, "MAPE": 0.5313252682015059, "median_APE": 0.5950676909916772, "p90_AE": 2.5935830370968915 }, "theta_D_K": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 3.3983782755291996, "MAPE": 0.5240601819517897, "median_APE": 0.4488409821267956, "p90_AE": 7.269358856755275 }, "v_sound_m_s": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 27.061803178085576, "MAPE": 0.39371279705030954, "median_APE": 0.18529466385634052, "p90_AE": 59.470405460335314 }, "density_g_cm3": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 0.05284119821879969, "MAPE": 0.5579850132036183, "median_APE": 0.6197556380982128, "p90_AE": 0.08907092203578948 }, "kappa_W_mK": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 0.0579913360020881, "MAPE": 0.4035025154620218, "median_APE": 0.18127218450544225, "p90_AE": 0.13689204742580685 } }, "aggregate": { "N_points": 50, "overall_MAPE": 0.48211715517384895, "overall_median_APE": 0.35376566112692953 }, "runtime": { "N_calls": 10, "mean_ms": 0.33507999614812434, "p95_ms": 0.37430005613714457 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 23.958449999999996, "MAPE": 8.026327439350975, "median_APE": 4.928695776255701, "p90_AE": 52.8835 }, "theta_D_K": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 89.8364, "MAPE": 13.785400840479957, "median_APE": 13.928409638554214, "p90_AE": 140.06849999999997 }, "v_sound_m_s": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 1916.6985, "MAPE": 29.02369313854333, "median_APE": 25.787674912346688, "p90_AE": 2559.21 }, "density_g_cm3": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 0.2784975000000001, "MAPE": 3.3914717188126082, "median_APE": 3.3707266515767604, "p90_AE": 0.38290000000000113 }, "kappa_W_mK": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 35.8299215, "MAPE": 237.61539635364633, "median_APE": 203.44595454545453, "p90_AE": 52.947950000000006 } }, "aggregate": { "N_points": 50, "overall_MAPE": 58.368457898166646, "overall_median_APE": 15.447265790020722 }, "runtime": { "N_calls": 10, "mean_ms": 0.003060000017285347, "p95_ms": 0.004999979864805937 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 36.341499999999996, "MAPE": 12.869745190358362, "median_APE": 11.358204575398247, "p90_AE": 56.629999999999995 }, "theta_D_K": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 0.5741000000000002, "MAPE": 9.606155746422415, "median_APE": 3.9095329067734084, "p90_AE": 1.1800000000000006 }, "kappa_W_mK": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 20, "overall_MAPE": 11.237950468390387, "overall_median_APE": 5.84351730802024 }, "runtime": { "N_calls": 10, "mean_ms": 0.002670002868399024, "p95_ms": 0.0033999676816165447 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 10, "N_pred": 10, "coverage_pct": 100.0, "MAE": 17.499999999999993, "MAPE": 6.01273559172781, "median_APE": 5.533104816886176, "p90_AE": 37.99999999999997 }, "theta_D_K": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 10, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 10, "overall_MAPE": 6.01273559172781, "overall_median_APE": 5.533104816886176 }, "runtime": { "N_calls": 10, "mean_ms": 0.0023399945348501205, "p95_ms": 0.0027999631129205227 } } ] }, { "fold_id": "family:category_oxide", "holdout_key": "category_oxide", "holdout_size": 4, "train_size": 176, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 0.5207236941224949, "MAPE": 0.2464184639826026, "median_APE": 0.18243745584679527, "p90_AE": 0.985078370182265 }, "theta_D_K": { "N_truth": 3, "N_pred": 2, "coverage_pct": 66.66666666666667, "MAE": 0.8443412679228857, "MAPE": 0.20284723183138453, "median_APE": 0.20284723183138453, "p90_AE": 1.5972772666186188 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 2, "coverage_pct": 66.66666666666667, "MAE": 20.80851125054801, "MAPE": 0.49177651746510503, "median_APE": 0.49177651746510503, "p90_AE": 35.51246799329738 }, "density_g_cm3": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 0.023698638143991424, "MAPE": 0.5605658702947556, "median_APE": 0.10122758365825937, "p90_AE": 0.059564902341208814 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 0.09126931715838123, "MAPE": 0.331174966003547, "median_APE": 0.13048158535738708, "p90_AE": 0.2525348845353754 } }, "aggregate": { "N_points": 13, "overall_MAPE": 0.36951733841813034, "overall_median_APE": 0.1689918061079948 }, "runtime": { "N_calls": 4, "mean_ms": 0.17994998779613525, "p95_ms": 0.24939997820183635 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 16.661500000000007, "MAPE": 8.498635518564184, "median_APE": 8.74186046511628, "p90_AE": 22.636500000000012 }, "theta_D_K": { "N_truth": 3, "N_pred": 2, "coverage_pct": 66.66666666666667, "MAE": 73.18100000000001, "MAPE": 18.272222222222226, "median_APE": 18.272222222222226, "p90_AE": 103.61400000000003 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 2, "coverage_pct": 66.66666666666667, "MAE": 1085.8999999999999, "MAPE": 23.141768292682926, "median_APE": 23.141768292682926, "p90_AE": 1296.1 }, "density_g_cm3": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 0.19832333333333332, "MAPE": 3.095663644487684, "median_APE": 3.1840136054421757, "p90_AE": 0.37051999999999996 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 3, "coverage_pct": 75.0, "MAE": 8.930951666666667, "MAPE": 58.41265972222222, "median_APE": 54.294333333333334, "p90_AE": 12.2362 } }, "aggregate": { "N_points": 13, "overall_MAPE": 22.52683520658712, "overall_median_APE": 11.87444444444444 }, "runtime": { "N_calls": 4, "mean_ms": 0.002900022082030773, "p95_ms": 0.004600035026669502 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 21.22, "MAPE": 9.574096302068483, "median_APE": 6.25701827242525, "p90_AE": 62.025000000000006 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.5511250000000001, "MAPE": 8.605863856285005, "median_APE": 8.784142771032469, "p90_AE": 1.3259999999999996 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 9.089980079176742, "overall_median_APE": 7.960419632969472 }, "runtime": { "N_calls": 4, "mean_ms": 0.0027500063879415393, "p95_ms": 0.0030999653972685337 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 25.125000000000007, "MAPE": 12.038190734231103, "median_APE": 8.513289036544839, "p90_AE": 64.5 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 4, "overall_MAPE": 12.038190734231103, "overall_median_APE": 8.513289036544839 }, "runtime": { "N_calls": 4, "mean_ms": 0.002574990503489971, "p95_ms": 0.0026999623514711857 } } ] }, { "fold_id": "family:category_ternary_oxide", "holdout_key": "category_ternary_oxide", "holdout_size": 3, "train_size": 177, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.8328189783338663, "MAPE": 0.4935469594352228, "median_APE": 0.5843102257253878, "p90_AE": 1.0400722017911903 }, "theta_D_K": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.9569630193562375, "MAPE": 0.2883360725321453, "median_APE": 0.35068580789019593, "p90_AE": 1.423525441350023 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 14.149109188137269, "MAPE": 0.3337750463106013, "median_APE": 0.3573274854987358, "p90_AE": 20.328297387756265 }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.10743411713020971, "MAPE": 1.8349168624355354, "median_APE": 1.6772194431523193, "p90_AE": 0.1276198949908789 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.012235615064934388, "MAPE": 0.17742937262925437, "median_APE": 0.17834397796545606, "p90_AE": 0.02389910987349886 } }, "aggregate": { "N_points": 15, "overall_MAPE": 0.6256008626685519, "overall_median_APE": 0.3573274854987358 }, "runtime": { "N_calls": 3, "mean_ms": 0.19253334418560067, "p95_ms": 0.21090003428980708 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 12.841, "MAPE": 6.727688185582128, "median_APE": 9.935121951219507, "p90_AE": 20.36699999999999 }, "theta_D_K": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 251.9228333333334, "MAPE": 73.2592363764348, "median_APE": 72.19441176470589, "p90_AE": 264.1165000000001 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 503.6349999999996, "MAPE": 12.842579262258198, "median_APE": 13.362564102564086, "p90_AE": 850.2299999999996 }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.29175833333333284, "MAPE": 4.861102194420379, "median_APE": 4.855859374999997, "p90_AE": 0.33839499999999934 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 8.917855, "MAPE": 206.81963286713287, "median_APE": 196.2723076923077, "p90_AE": 12.7577 } }, "aggregate": { "N_points": 15, "overall_MAPE": 60.90204777716567, "overall_median_APE": 13.362564102564086 }, "runtime": { "N_calls": 3, "mean_ms": 0.0039000296965241432, "p95_ms": 0.00500003807246685 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 5.1633333333333455, "MAPE": 3.0492329133737814, "median_APE": 3.317073170731713, "p90_AE": 8.330000000000013 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.15133333333333296, "MAPE": 2.4782936813581133, "median_APE": 2.3646723646723578, "p90_AE": 0.1899999999999995 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 6, "overall_MAPE": 2.7637632973659474, "overall_median_APE": 2.7604092720371716 }, "runtime": { "N_calls": 3, "mean_ms": 0.0029333362666269145, "p95_ms": 0.003300025127828121 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 23.999999999999982, "MAPE": 14.772021953441511, "median_APE": 8.292682926829228, "p90_AE": 45.00000000000003 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 3, "overall_MAPE": 14.772021953441511, "overall_median_APE": 8.292682926829228 }, "runtime": { "N_calls": 3, "mean_ms": 0.0026666869719823203, "p95_ms": 0.002700020559132099 } } ] }, { "fold_id": "family:category_tm_pnictide", "holdout_key": "category_tm_pnictide", "holdout_size": 4, "train_size": 176, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 2.2283869040103284, "MAPE": 0.7626493693200957, "median_APE": 0.7735824224500814, "p90_AE": 5.148743358273066 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 5.192303298500306, "MAPE": 0.9428850893838725, "median_APE": 0.9420073988922117, "p90_AE": 8.28373467740846 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 50.37424474767499, "MAPE": 0.705563708875238, "median_APE": 0.2093557521836164, "p90_AE": 173.23719159018674 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.037118868875207545, "MAPE": 0.5758526448439489, "median_APE": 0.5633200126488421, "p90_AE": 0.06498121797973333 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.0512046349392703, "MAPE": 0.41392959618654485, "median_APE": 0.4075477186976324, "p90_AE": 0.08939447853641269 } }, "aggregate": { "N_points": 20, "overall_MAPE": 0.6801760817219398, "overall_median_APE": 0.4513785368359624 }, "runtime": { "N_calls": 4, "mean_ms": 0.3342749987496063, "p95_ms": 0.35180000122636557 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 37.584500000000006, "MAPE": 13.869623950310976, "median_APE": 11.44350638113779, "p90_AE": 65.10300000000001 }, "theta_D_K": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 124.52425, "MAPE": 19.3169804751642, "median_APE": 14.177096726190474, "p90_AE": 288.756 }, "v_sound_m_s": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 1066.4837499999999, "MAPE": 17.511885651527344, "median_APE": 16.169364636830515, "p90_AE": 1790.705 }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.3709637499999998, "MAPE": 5.891593510687808, "median_APE": 3.1044116681440617, "p90_AE": 1.0359049999999996 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 43.310275000000004, "MAPE": 337.0164479166667, "median_APE": 288.2727916666667, "p90_AE": 66.8914 } }, "aggregate": { "N_points": 20, "overall_MAPE": 78.7213063008714, "overall_median_APE": 14.550605020779509 }, "runtime": { "N_calls": 4, "mean_ms": 0.003575027221813798, "p95_ms": 0.004900037311017513 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 22.828750000000014, "MAPE": 8.826543637233687, "median_APE": 5.3837491823194155, "p90_AE": 62.0 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 0.213125, "MAPE": 2.4050022277016225, "median_APE": 1.8498801667815776, "p90_AE": 0.47199999999999953 }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 5.615772932467655, "overall_median_APE": 3.790196393644671 }, "runtime": { "N_calls": 4, "mean_ms": 0.0028750073397532105, "p95_ms": 0.003500026650726795 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 4, "N_pred": 4, "coverage_pct": 100.0, "MAE": 41.875000000000036, "MAPE": 15.439996450021575, "median_APE": 13.988838913876405, "p90_AE": 84.00000000000011 }, "theta_D_K": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 4, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 4, "overall_MAPE": 15.439996450021575, "overall_median_APE": 13.988838913876405 }, "runtime": { "N_calls": 4, "mean_ms": 0.002625005436129868, "p95_ms": 0.002800021320581436 } } ] }, { "fold_id": "family:fcc_metals", "holdout_key": "fcc_metals", "holdout_size": 8, "train_size": 172, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 1.545169230331469, "MAPE": 0.6507736821457638, "median_APE": 0.32418667538800156, "p90_AE": 6.243907659490446 }, "theta_D_K": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 1.2951349026634098, "MAPE": 0.39311421030901567, "median_APE": 0.3794453361591479, "p90_AE": 2.3472923230021934 }, "v_sound_m_s": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 38.233259181107485, "MAPE": 0.8359871443176654, "median_APE": 0.6330894190181053, "p90_AE": 93.90545591473983 }, "density_g_cm3": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 0.14018372177975835, "MAPE": 0.8909090986772921, "median_APE": 0.34643609282436016, "p90_AE": 0.5873374890654759 }, "kappa_W_mK": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 1.4171798940375755, "MAPE": 0.5638445027720559, "median_APE": 0.3694518483773798, "p90_AE": 6.631517761600094 } }, "aggregate": { "N_points": 40, "overall_MAPE": 0.6669257276443585, "overall_median_APE": 0.39471409669434054 }, "runtime": { "N_calls": 8, "mean_ms": 0.44411250564735383, "p95_ms": 0.48590003279969096 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 8, "N_pred": 1, "coverage_pct": 12.5, "MAE": 26.599500000000006, "MAPE": 25.576442307692314, "median_APE": 25.576442307692314, "p90_AE": 26.599500000000006 }, "theta_D_K": { "N_truth": 8, "N_pred": 1, "coverage_pct": 12.5, "MAE": 33.087999999999994, "MAPE": 14.705777777777776, "median_APE": 14.705777777777776, "p90_AE": 33.087999999999994 }, "v_sound_m_s": { "N_truth": 8, "N_pred": 1, "coverage_pct": 12.5, "MAE": 1964.13, "MAPE": 53.81178082191781, "median_APE": 53.81178082191781, "p90_AE": 1964.13 }, "density_g_cm3": { "N_truth": 8, "N_pred": 1, "coverage_pct": 12.5, "MAE": 0.4773999999999994, "MAPE": 4.55100095328884, "median_APE": 4.55100095328884, "p90_AE": 0.4773999999999994 }, "kappa_W_mK": { "N_truth": 8, "N_pred": 1, "coverage_pct": 12.5, "MAE": 427.68104, "MAPE": 99.69255011655011, "median_APE": 99.69255011655011, "p90_AE": 427.68104 } }, "aggregate": { "N_points": 5, "overall_MAPE": 39.66751039544537, "overall_median_APE": 25.576442307692314 }, "runtime": { "N_calls": 8, "mean_ms": 0.001037493348121643, "p95_ms": 0.005099980626255274 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 11.349374999999995, "MAPE": 5.466209991487219, "median_APE": 4.5704094292803905, "p90_AE": 26.400000000000006 }, "theta_D_K": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 0.4844375000000001, "MAPE": 2.971714061241438, "median_APE": 3.013901283897535, "p90_AE": 1.0565000000000033 }, "kappa_W_mK": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 16, "overall_MAPE": 4.218962026364329, "overall_median_APE": 3.5253682487725015 }, "runtime": { "N_calls": 8, "mean_ms": 0.0027874993975274265, "p95_ms": 0.0035999692045152187 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 18.749999999999982, "MAPE": 9.926325897050644, "median_APE": 8.5495172635126, "p90_AE": 35.0 }, "theta_D_K": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 8, "overall_MAPE": 9.926325897050644, "overall_median_APE": 8.5495172635126 }, "runtime": { "N_calls": 8, "mean_ms": 0.0024874971131794155, "p95_ms": 0.002800021320581436 } } ] }, { "fold_id": "family:group14_chalcogenides", "holdout_key": "group14_chalcogenides", "holdout_size": 7, "train_size": 173, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.22055157016039675, "MAPE": 0.5513818494520261, "median_APE": 0.2654226646413349, "p90_AE": 0.8996956914278158 }, "theta_D_K": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.1477280332376145, "MAPE": 0.704823883878513, "median_APE": 0.5481834814506628, "p90_AE": 2.013813191548735 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 24.758637639312642, "MAPE": 0.8995856014355132, "median_APE": 0.828509597652974, "p90_AE": 43.462974516120994 }, "density_g_cm3": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.0258523106997471, "MAPE": 0.39165472018683195, "median_APE": 0.37824661192147985, "p90_AE": 0.04773880070671055 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 0.008311951575255947, "MAPE": 0.3323134582920323, "median_APE": 0.16481250213640664, "p90_AE": 0.01967732302722469 } }, "aggregate": { "N_points": 34, "overall_MAPE": 0.5813724080155172, "overall_median_APE": 0.4840823578552772 }, "runtime": { "N_calls": 7, "mean_ms": 0.37062856634812696, "p95_ms": 0.40009996155276895 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 6.579371428571428, "MAPE": 18.09224735066648, "median_APE": 4.8758510638297965, "p90_AE": 16.927799999999998 }, "theta_D_K": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 47.43085714285714, "MAPE": 27.426498912191384, "median_APE": 32.581904761904774, "p90_AE": 94.457 }, "v_sound_m_s": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 797.107857142857, "MAPE": 28.244062208787902, "median_APE": 30.643518518518526, "p90_AE": 1546.4049999999997 }, "density_g_cm3": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.27152333333333295, "MAPE": 4.232668945624668, "median_APE": 4.502616534067724, "p90_AE": 0.4141899999999996 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.7066187857142856, "MAPE": 58.532731625258805, "median_APE": 22.903249999999996, "p90_AE": 4.21949 } }, "aggregate": { "N_points": 34, "overall_MAPE": 27.984258657414127, "overall_median_APE": 15.270766045548656 }, "runtime": { "N_calls": 7, "mean_ms": 0.003128570304917438, "p95_ms": 0.004999979864805937 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 1.789285714285713, "MAPE": 4.3121366035435225, "median_APE": 4.8095238095238, "p90_AE": 3.3699999999999974 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.2358333333333332, "MAPE": 3.71519189915027, "median_APE": 4.208248664107767, "p90_AE": 0.29000000000000004 }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 13, "overall_MAPE": 4.036623663054329, "overall_median_APE": 4.579288025889959 }, "runtime": { "N_calls": 7, "mean_ms": 0.0027142814360558987, "p95_ms": 0.003300025127828121 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 7, "N_pred": 7, "coverage_pct": 100.0, "MAE": 3.7857142857142887, "MAPE": 10.269799457804856, "median_APE": 4.761904761904779, "p90_AE": 10.000000000000004 }, "theta_D_K": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 7, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 7, "overall_MAPE": 10.269799457804856, "overall_median_APE": 4.761904761904779 }, "runtime": { "N_calls": 7, "mean_ms": 0.002471430759344782, "p95_ms": 0.002900022082030773 } } ] }, { "fold_id": "family:group14_oxides", "holdout_key": "group14_oxides", "holdout_size": 3, "train_size": 177, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.4588536667735094, "MAPE": 0.3344631214503126, "median_APE": 0.4569505803911061, "p90_AE": 1.016681949006795 }, "theta_D_K": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.8389537972706288, "MAPE": 0.13165202164658282, "median_APE": 0.06798191036479694, "p90_AE": 1.9688454648114657 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 2.79404874769034, "MAPE": 0.06914971050373879, "median_APE": 0.07210657642930789, "p90_AE": 4.795087332548974 }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.06219939023038871, "MAPE": 0.8019300411890103, "median_APE": 0.7827214987098605, "p90_AE": 0.08956330016953018 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.030396317999271005, "MAPE": 0.14460047846087995, "median_APE": 0.1236328306395735, "p90_AE": 0.07791990184813358 } }, "aggregate": { "N_points": 15, "overall_MAPE": 0.2963590746501049, "overall_median_APE": 0.1236328306395735 }, "runtime": { "N_calls": 3, "mean_ms": 0.36743335658684373, "p95_ms": 0.3880000440403819 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 35.8952, "MAPE": 29.32360258802237, "median_APE": 22.344339622641513, "p90_AE": 47.370000000000005 }, "theta_D_K": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 92.07166666666667, "MAPE": 23.051086673038125, "median_APE": 25.5288, "p90_AE": 159.555 }, "v_sound_m_s": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 1737.96, "MAPE": 36.318699202196385, "median_APE": 41.82929133858268, "p90_AE": 2656.16 }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.5710099999999999, "MAPE": 6.7519426706596635, "median_APE": 4.592517985611514, "p90_AE": 1.206339999999999 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 12.017096666666665, "MAPE": 37.89489523809524, "median_APE": 26.219285714285718, "p90_AE": 34.130359999999996 } }, "aggregate": { "N_points": 15, "overall_MAPE": 26.668045274402353, "overall_median_APE": 24.86045112781955 }, "runtime": { "N_calls": 3, "mean_ms": 0.003833362522224585, "p95_ms": 0.005100038833916187 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 93.27500000000002, "MAPE": 45.87353424872555, "median_APE": 46.83018867924528, "p90_AE": 170.4 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 3, "N_pred": 3, "coverage_pct": 100.0, "MAE": 0.5739999999999998, "MAPE": 8.509927412013402, "median_APE": 7.755395683453245, "p90_AE": 0.9690000000000003 }, "kappa_W_mK": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 6, "overall_MAPE": 27.19173083036948, "overall_median_APE": 20.136374296435278 }, "runtime": { "N_calls": 3, "mean_ms": 0.0028666690923273563, "p95_ms": 0.003300025127828121 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 3, "N_pred": 2, "coverage_pct": 66.66666666666667, "MAE": 96.00000000000001, "MAPE": 39.564136317098146, "median_APE": 39.564136317098146, "p90_AE": 136.00000000000003 }, "theta_D_K": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 3, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 2, "overall_MAPE": 39.564136317098146, "overall_median_APE": 39.564136317098146 }, "runtime": { "N_calls": 3, "mean_ms": 0.0020333294135828814, "p95_ms": 0.002700020559132099 } } ] }, { "fold_id": "family:hcp_metals", "holdout_key": "hcp_metals", "holdout_size": 11, "train_size": 169, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 0.8324509563763258, "MAPE": 0.394489982772126, "median_APE": 0.37793464182167275, "p90_AE": 1.7189482232327862 }, "theta_D_K": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 4.310024838493996, "MAPE": 1.1874189864305382, "median_APE": 0.9018550712106995, "p90_AE": 7.478394451672898 }, "v_sound_m_s": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 34.42811704227093, "MAPE": 0.700407090042826, "median_APE": 0.311952760076989, "p90_AE": 81.49425098479605 }, "density_g_cm3": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 0.16904274027186156, "MAPE": 1.9895272450958223, "median_APE": 1.3507204692220758, "p90_AE": 0.38071885627264024 }, "kappa_W_mK": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 0.23104823672323088, "MAPE": 0.5284982502421, "median_APE": 0.5280879167307068, "p90_AE": 0.464717366723022 } }, "aggregate": { "N_points": 55, "overall_MAPE": 0.9600683109166824, "overall_median_APE": 0.5119544503824761 }, "runtime": { "N_calls": 11, "mean_ms": 0.4264909128489142, "p95_ms": 0.45170000521466136 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 11, "N_pred": 8, "coverage_pct": 72.72727272727273, "MAE": 3.555737499999995, "MAPE": 1.6926837996515351, "median_APE": 1.6621951219512228, "p90_AE": 11.555000000000007 }, "theta_D_K": { "N_truth": 11, "N_pred": 8, "coverage_pct": 72.72727272727273, "MAE": 38.10799999999999, "MAPE": 9.153653319167113, "median_APE": 7.977385714285708, "p90_AE": 75.53050000000002 }, "v_sound_m_s": { "N_truth": 11, "N_pred": 8, "coverage_pct": 72.72727272727273, "MAE": 1926.698125, "MAPE": 36.91019428579713, "median_APE": 37.973485888068836, "p90_AE": 2388.7 }, "density_g_cm3": { "N_truth": 11, "N_pred": 8, "coverage_pct": 72.72727272727273, "MAE": 0.20460562500000012, "MAPE": 2.3078205578827196, "median_APE": 1.6669792050006125, "p90_AE": 0.5424000000000007 }, "kappa_W_mK": { "N_truth": 11, "N_pred": 8, "coverage_pct": 72.72727272727273, "MAE": 19.064543750000002, "MAPE": 33.212303896883704, "median_APE": 31.512826086956515, "p90_AE": 74.2476 } }, "aggregate": { "N_points": 40, "overall_MAPE": 16.65533117187644, "overall_median_APE": 7.977385714285708 }, "runtime": { "N_calls": 11, "mean_ms": 0.0025181851180439644, "p95_ms": 0.004800036549568176 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 8.460909090909091, "MAPE": 4.900711954151544, "median_APE": 3.095015576323988, "p90_AE": 27.370000000000005 }, "theta_D_K": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 0.28754545454545444, "MAPE": 2.831147066539223, "median_APE": 2.7711703139866777, "p90_AE": 0.5824999999999996 }, "kappa_W_mK": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 22, "overall_MAPE": 3.8659295103453832, "overall_median_APE": 2.8525274882193425 }, "runtime": { "N_calls": 11, "mean_ms": 0.002736368597569791, "p95_ms": 0.003500026650726795 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 11, "N_pred": 11, "coverage_pct": 100.0, "MAE": 7.590909090909078, "MAPE": 4.71588602457407, "median_APE": 3.2296650717703352, "p90_AE": 13.5 }, "theta_D_K": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 11, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 11, "overall_MAPE": 4.71588602457407, "overall_median_APE": 3.2296650717703352 }, "runtime": { "N_calls": 11, "mean_ms": 0.002454537686637857, "p95_ms": 0.002800021320581436 } } ] }, { "fold_id": "family:iii_v_semiconductors", "holdout_key": "iii_v_semiconductors", "holdout_size": 13, "train_size": 167, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 0.8398232385688572, "MAPE": 0.6594880606274695, "median_APE": 0.6668226199642905, "p90_AE": 1.9788405088311833 }, "theta_D_K": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 6.733635339480504, "MAPE": 1.160967901864664, "median_APE": 0.829777427667705, "p90_AE": 16.453626509648984 }, "v_sound_m_s": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 41.196974856726264, "MAPE": 0.6838079450149916, "median_APE": 0.6409232665658859, "p90_AE": 58.13515060954796 }, "density_g_cm3": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 0.026793985142634424, "MAPE": 0.6572797903427252, "median_APE": 0.7937559249499168, "p90_AE": 0.061041226656319836 }, "kappa_W_mK": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 2.321982874715604, "MAPE": 0.8021863262517018, "median_APE": 0.5207189498670878, "p90_AE": 4.74644048209035 } }, "aggregate": { "N_points": 60, "overall_MAPE": 0.8040348560267758, "overall_median_APE": 0.676911064689683 }, "runtime": { "N_calls": 13, "mean_ms": 0.33724615171265143, "p95_ms": 0.387999985832721 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 31.12003076923077, "MAPE": 16.767033263855595, "median_APE": 14.325492957746484, "p90_AE": 33.13749999999999 }, "theta_D_K": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 102.9093076923077, "MAPE": 10.91732560921026, "median_APE": 8.117766497461933, "p90_AE": 244.45899999999995 }, "v_sound_m_s": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 2543.596153846154, "MAPE": 37.38492784910018, "median_APE": 37.841029411764715, "p90_AE": 4317.32 }, "density_g_cm3": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 0.28472437500000003, "MAPE": 7.704181272229297, "median_APE": 3.221382113821129, "p90_AE": 1.51643 }, "kappa_W_mK": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 137.72816192307693, "MAPE": 87.13610674785818, "median_APE": 86.36385185185185, "p90_AE": 264.34135000000003 } }, "aggregate": { "N_points": 60, "overall_MAPE": 34.00505942146915, "overall_median_APE": 18.349198394923476 }, "runtime": { "N_calls": 13, "mean_ms": 0.0028538410193645037, "p95_ms": 0.005199981387704611 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 21.315384615384616, "MAPE": 13.708268698067096, "median_APE": 14.847619047619052, "p90_AE": 32.25999999999999 }, "theta_D_K": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 8, "N_pred": 8, "coverage_pct": 100.0, "MAE": 0.3529375000000001, "MAPE": 9.16316667917765, "median_APE": 5.963284137448809, "p90_AE": 1.1800000000000002 }, "kappa_W_mK": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 21, "overall_MAPE": 11.976801262299686, "overall_median_APE": 7.541666666666669 }, "runtime": { "N_calls": 13, "mean_ms": 0.0025153903362269583, "p95_ms": 0.0033999676816165447 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 13, "N_pred": 13, "coverage_pct": 100.0, "MAE": 32.538461538461554, "MAPE": 17.740918497465486, "median_APE": 15.517241379310356, "p90_AE": 35.5 }, "theta_D_K": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 8, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 13, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 13, "overall_MAPE": 17.740918497465486, "overall_median_APE": 15.517241379310356 }, "runtime": { "N_calls": 13, "mean_ms": 0.0023230677470564842, "p95_ms": 0.00300002284348011 } } ] }, { "fold_id": "family:tm_oxides", "holdout_key": "tm_oxides", "holdout_size": 27, "train_size": 153, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 27, "N_pred": 21, "coverage_pct": 77.77777777777777, "MAE": 4.017232504436569, "MAPE": 2.4584314194740307, "median_APE": 1.8870593071401203, "p90_AE": 8.033599511549909 }, "theta_D_K": { "N_truth": 27, "N_pred": 21, "coverage_pct": 77.77777777777777, "MAE": 5.511490785286169, "MAPE": 1.2864808521005535, "median_APE": 0.9902699982471586, "p90_AE": 10.191695072075959 }, "v_sound_m_s": { "N_truth": 27, "N_pred": 21, "coverage_pct": 77.77777777777777, "MAE": 183.96397725600556, "MAPE": 4.555812732172685, "median_APE": 3.1019697878212087, "p90_AE": 442.0469318202695 }, "density_g_cm3": { "N_truth": 27, "N_pred": 21, "coverage_pct": 77.77777777777777, "MAE": 0.10779635891493458, "MAPE": 1.8684861249227576, "median_APE": 1.4962328519027863, "p90_AE": 0.20660062218947228 }, "kappa_W_mK": { "N_truth": 27, "N_pred": 21, "coverage_pct": 77.77777777777777, "MAE": 0.24617924259260007, "MAPE": 1.759826861092321, "median_APE": 1.1245707736515596, "p90_AE": 0.5883286063638469 } }, "aggregate": { "N_points": 105, "overall_MAPE": 2.3858075979524687, "overall_median_APE": 1.546249685716461 }, "runtime": { "N_calls": 27, "mean_ms": 0.33833704569756434, "p95_ms": 0.4521000082604587 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 27, "N_pred": 20, "coverage_pct": 74.07407407407408, "MAE": 31.764574999999997, "MAPE": 29.18598855552719, "median_APE": 13.444410782442743, "p90_AE": 62.703 }, "theta_D_K": { "N_truth": 27, "N_pred": 20, "coverage_pct": 74.07407407407408, "MAE": 80.74802499999998, "MAPE": 19.64796607345856, "median_APE": 14.575866666666666, "p90_AE": 177.94600000000003 }, "v_sound_m_s": { "N_truth": 27, "N_pred": 20, "coverage_pct": 74.07407407407408, "MAE": 1192.5557499999995, "MAPE": 25.40269696240716, "median_APE": 27.17800861062463, "p90_AE": 2087.82 }, "density_g_cm3": { "N_truth": 27, "N_pred": 20, "coverage_pct": 74.07407407407408, "MAE": 0.42269299999999993, "MAPE": 6.3310171867756875, "median_APE": 5.459855253573139, "p90_AE": 0.8683299999999994 }, "kappa_W_mK": { "N_truth": 27, "N_pred": 20, "coverage_pct": 74.07407407407408, "MAE": 9.671411925, "MAPE": 263.31549309385235, "median_APE": 83.00283974358975, "p90_AE": 16.257385 } }, "aggregate": { "N_points": 100, "overall_MAPE": 68.77663237440419, "overall_median_APE": 16.221376470588233 }, "runtime": { "N_calls": 27, "mean_ms": 0.0021370403744556286, "p95_ms": 0.003100023604929447 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 27, "N_pred": 25, "coverage_pct": 92.5925925925926, "MAE": 27.839799999999997, "MAPE": 23.696617793468334, "median_APE": 12.083969465648854, "p90_AE": 44.5 }, "theta_D_K": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 27, "N_pred": 26, "coverage_pct": 96.29629629629629, "MAE": 0.4655384615384615, "MAPE": 6.913299405973659, "median_APE": 5.556613258342965, "p90_AE": 0.8790000000000013 }, "kappa_W_mK": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 51, "overall_MAPE": 15.140416262588696, "overall_median_APE": 8.47637415621988 }, "runtime": { "N_calls": 27, "mean_ms": 0.0023074055745921753, "p95_ms": 0.002900022082030773 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 27, "N_pred": 27, "coverage_pct": 100.0, "MAE": 28.370370370370356, "MAPE": 23.85879349604886, "median_APE": 15.769230769230768, "p90_AE": 60.50000000000003 }, "theta_D_K": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 27, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 27, "overall_MAPE": 23.85879349604886, "overall_median_APE": 15.769230769230768 }, "runtime": { "N_calls": 27, "mean_ms": 0.0021999995051710693, "p95_ms": 0.0024999608285725117 } } ] }, { "fold_id": "family:tm_sulfides", "holdout_key": "tm_sulfides", "holdout_size": 6, "train_size": 174, "models": [ { "model": "flux", "properties": { "B_GPa": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.5377015399031334, "MAPE": 0.8374417490852774, "median_APE": 0.3972809395674517, "p90_AE": 1.8593238982438294 }, "theta_D_K": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 2.5110804870834613, "MAPE": 0.8155436590691449, "median_APE": 0.4414405094896149, "p90_AE": 7.579736618322215 }, "v_sound_m_s": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 169.02682550292744, "MAPE": 4.006069673655907, "median_APE": 2.3481575386442723, "p90_AE": 599.7233231568994 }, "density_g_cm3": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.0677082857210829, "MAPE": 1.266223027271357, "median_APE": 1.0746815165423733, "p90_AE": 0.14504893985690703 }, "kappa_W_mK": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.5009165987683525, "MAPE": 3.0051300898688247, "median_APE": 2.0179462417695615, "p90_AE": 0.9942763544736124 } }, "aggregate": { "N_points": 30, "overall_MAPE": 1.986081639790102, "overall_median_APE": 0.7784286794761789 }, "runtime": { "N_calls": 6, "mean_ms": 0.4166500099624197, "p95_ms": 0.4694000235758722 } }, { "model": "aflow", "properties": { "B_GPa": { "N_truth": 6, "N_pred": 4, "coverage_pct": 66.66666666666667, "MAE": 19.117824999999996, "MAPE": 21.8351056944469, "median_APE": 17.92311903680145, "p90_AE": 36.82899999999999 }, "theta_D_K": { "N_truth": 6, "N_pred": 4, "coverage_pct": 66.66666666666667, "MAE": 21.24325000000001, "MAPE": 6.097440347084124, "median_APE": 4.679392544570506, "p90_AE": 39.031000000000006 }, "v_sound_m_s": { "N_truth": 6, "N_pred": 4, "coverage_pct": 66.66666666666667, "MAE": 1406.64, "MAPE": 31.822230671220996, "median_APE": 33.73361635220126, "p90_AE": 2121.87 }, "density_g_cm3": { "N_truth": 6, "N_pred": 4, "coverage_pct": 66.66666666666667, "MAE": 0.2928725000000001, "MAPE": 4.95859691789508, "median_APE": 4.813934993084377, "p90_AE": 0.61327 }, "kappa_W_mK": { "N_truth": 6, "N_pred": 4, "coverage_pct": 66.66666666666667, "MAE": 19.763802499999997, "MAPE": 72.89273977102103, "median_APE": 77.8856739864865, "p90_AE": 26.7299 } }, "aggregate": { "N_points": 20, "overall_MAPE": 27.521222680333626, "overall_median_APE": 11.150558115719406 }, "runtime": { "N_calls": 6, "mean_ms": 0.00236667498635749, "p95_ms": 0.004699977580457926 } }, { "model": "jarvis", "properties": { "B_GPa": { "N_truth": 6, "N_pred": 5, "coverage_pct": 83.33333333333333, "MAE": 14.850000000000003, "MAPE": 19.517693449903085, "median_APE": 19.429577464788736, "p90_AE": 27.590000000000003 }, "theta_D_K": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 0.5030833333333334, "MAPE": 8.282753463087479, "median_APE": 5.039821289821289, "p90_AE": 1.6980000000000004 }, "kappa_W_mK": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 11, "overall_MAPE": 13.389544366185481, "overall_median_APE": 12.215568862275452 }, "runtime": { "N_calls": 6, "mean_ms": 0.002683314960449934, "p95_ms": 0.0030999653972685337 } }, { "model": "matbench", "properties": { "B_GPa": { "N_truth": 6, "N_pred": 6, "coverage_pct": 100.0, "MAE": 17.75000000000001, "MAPE": 28.760362081670618, "median_APE": 24.93047830923249, "p90_AE": 36.0 }, "theta_D_K": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 6, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 6, "overall_MAPE": 28.760362081670618, "overall_median_APE": 24.93047830923249 }, "runtime": { "N_calls": 6, "mean_ms": 0.0022999884095042944, "p95_ms": 0.002500019036233425 } } ] } ], "model_summary": { "flux": { "properties": { "B_GPa": { "N_truth": 171, "N_pred": 164, "coverage_pct": 95.90643274853801, "MAE": 1.1675261625357083, "MAPE": 0.9055631591475641, "median_APE": null, "p90_AE": null }, "theta_D_K": { "N_truth": 170, "N_pred": 163, "coverage_pct": 95.88235294117646, "MAE": 5.205456723521288, "MAPE": 1.3283115919723314, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 170, "N_pred": 163, "coverage_pct": 95.88235294117646, "MAE": 64.99921138899853, "MAPE": 1.4405422921045707, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 165, "N_pred": 158, "coverage_pct": 95.75757575757575, "MAE": 0.07221132944959033, "MAPE": 1.223072650386257, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 156, "N_pred": 149, "coverage_pct": 95.51282051282051, "MAE": 0.643683120534908, "MAPE": 0.9185318092827579, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 797, "overall_MAPE": 1.1668033369496802, "overall_median_APE": null }, "runtime_mean_ms_across_folds": 0.3346740560111419 }, "aflow": { "properties": { "B_GPa": { "N_truth": 171, "N_pred": 142, "coverage_pct": 83.04093567251462, "MAE": 15.90960161971831, "MAPE": 14.358000548605853, "median_APE": null, "p90_AE": null }, "theta_D_K": { "N_truth": 170, "N_pred": 141, "coverage_pct": 82.94117647058823, "MAE": 67.04219929078013, "MAPE": 15.679016250064555, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 170, "N_pred": 141, "coverage_pct": 82.94117647058823, "MAE": 1449.4992340425533, "MAPE": 29.509083192618583, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 165, "N_pred": 136, "coverage_pct": 82.42424242424242, "MAE": 0.2997607647058823, "MAPE": 5.569458936552751, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 156, "N_pred": 127, "coverage_pct": 81.41025641025641, "MAE": 55.624323086614176, "MAPE": 122.92769695814138, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 687, "overall_MAPE": 36.069318818596045, "overall_median_APE": null }, "runtime_mean_ms_across_folds": 0.0028987191015071253 }, "jarvis": { "properties": { "B_GPa": { "N_truth": 171, "N_pred": 163, "coverage_pct": 95.32163742690058, "MAE": 18.06831288343558, "MAPE": 15.419080397690172, "median_APE": null, "p90_AE": null }, "theta_D_K": { "N_truth": 170, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 170, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 165, "N_pred": 160, "coverage_pct": 96.96969696969697, "MAE": 0.349190625, "MAPE": 6.342549010398218, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 156, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 323, "overall_MAPE": 10.922965778598183, "overall_median_APE": null }, "runtime_mean_ms_across_folds": 0.0027059151809150435 }, "matbench": { "properties": { "B_GPa": { "N_truth": 171, "N_pred": 170, "coverage_pct": 99.41520467836257, "MAE": 19.803529411764707, "MAPE": 18.421377776809848, "median_APE": null, "p90_AE": null }, "theta_D_K": { "N_truth": 170, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "v_sound_m_s": { "N_truth": 170, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "density_g_cm3": { "N_truth": 165, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null }, "kappa_W_mK": { "N_truth": 156, "N_pred": 0, "coverage_pct": 0.0, "MAE": null, "MAPE": null, "median_APE": null, "p90_AE": null } }, "aggregate": { "N_points": 170, "overall_MAPE": 18.421377776809848, "overall_median_APE": null }, "runtime_mean_ms_across_folds": 0.0024547547337159907 } } }