diff --git "a/data/benchmarks/bfcl.json" "b/data/benchmarks/bfcl.json" deleted file mode 100644--- "a/data/benchmarks/bfcl.json" +++ /dev/null @@ -1,4117 +0,0 @@ -{ - "models": [ - { - "model_id": "amazon/amazon-nova-2-lite-v1-0-fc", - "name": "Amazon-Nova-2-Lite-v1:0 (FC)", - "developer": "amazon", - "scores": { - "bfcl.overall.rank": 80.0, - "bfcl.overall.overall_accuracy": 27.1, - "bfcl.overall.total_cost_usd": 78.19, - "bfcl.overall.latency_mean_s": 8.55, - "bfcl.overall.latency_std_s": 9.85, - "bfcl.overall.latency_p95_s": 27.62, - "bfcl.non_live.ast_accuracy": 86.96, - "bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl.live.live_accuracy": 80.83, - "bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl.live.live_multiple_ast_accuracy": 80.15, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 2.12, - "bfcl.multi_turn.base_accuracy": 2.5, - "bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl.web_search.accuracy": 5.0, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl.memory.accuracy": 2.37, - "bfcl.memory.kv_accuracy": 1.94, - "bfcl.memory.vector_accuracy": 3.23, - "bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 82.11 - } - }, - { - "model_id": "amazon/amazon-nova-micro-v1-0-fc", - "name": "Amazon-Nova-Micro-v1:0 (FC)", - "developer": "amazon", - "scores": { - "bfcl.overall.rank": 95.0, - "bfcl.overall.overall_accuracy": 22.29, - "bfcl.overall.total_cost_usd": 1.81, - "bfcl.overall.latency_mean_s": 1.12, - "bfcl.overall.latency_std_s": 0.45, - "bfcl.overall.latency_p95_s": 1.79, - "bfcl.non_live.ast_accuracy": 74.1, - "bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl.non_live.multiple_ast_accuracy": 87.5, - "bfcl.non_live.parallel_ast_accuracy": 75.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 62.5, - "bfcl.live.live_accuracy": 66.32, - "bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl.live.live_multiple_ast_accuracy": 64.96, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 1.38, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl.web_search.accuracy": 1.5, - "bfcl.web_search.base_accuracy": 3.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 2.37, - "bfcl.memory.kv_accuracy": 1.94, - "bfcl.memory.vector_accuracy": 2.58, - "bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 70.65 - } - }, - { - "model_id": "amazon/amazon-nova-pro-v1-0-fc", - "name": "Amazon-Nova-Pro-v1:0 (FC)", - "developer": "amazon", - "scores": { - "bfcl.overall.rank": 88.0, - "bfcl.overall.overall_accuracy": 24.97, - "bfcl.overall.total_cost_usd": 48.44, - "bfcl.overall.latency_mean_s": 2.25, - "bfcl.overall.latency_std_s": 1.91, - "bfcl.overall.latency_p95_s": 3.29, - "bfcl.non_live.ast_accuracy": 86.58, - "bfcl.non_live.simple_ast_accuracy": 75.83, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 84.0, - "bfcl.live.live_accuracy": 78.53, - "bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 1.88, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl.web_search.accuracy": 2.5, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 1.94, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 1.29, - "bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 70.06 - } - }, - { - "model_id": "anthropic/claude-haiku-4-5-20251001-fc", - "name": "Claude-Haiku-4-5-20251001 (FC)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 6.0, - "bfcl.overall.overall_accuracy": 68.7, - "bfcl.overall.total_cost_usd": 14.23, - "bfcl.overall.latency_mean_s": 1.68, - "bfcl.overall.latency_std_s": 3.92, - "bfcl.overall.latency_p95_s": 3.15, - "bfcl.non_live.ast_accuracy": 86.5, - "bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 78.68, - "bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl.live.live_multiple_ast_accuracy": 77.59, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 53.62, - "bfcl.multi_turn.base_accuracy": 63.5, - "bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl.multi_turn.long_context_accuracy": 56.0, - "bfcl.web_search.accuracy": 83.5, - "bfcl.web_search.base_accuracy": 86.0, - "bfcl.web_search.no_snippet_accuracy": 81.0, - "bfcl.memory.accuracy": 54.41, - "bfcl.memory.kv_accuracy": 51.61, - "bfcl.memory.vector_accuracy": 55.48, - "bfcl.memory.recursive_summarization_accuracy": 56.13, - "bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl.relevance.irrelevance_detection_accuracy": 85.11 - } - }, - { - "model_id": "anthropic/claude-haiku-4-5-20251001-prompt", - "name": "Claude-Haiku-4-5-20251001 (Prompt)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 87.0, - "bfcl.overall.overall_accuracy": 25.26, - "bfcl.overall.total_cost_usd": 45.13, - "bfcl.overall.latency_mean_s": 3.75, - "bfcl.overall.latency_std_s": 19.96, - "bfcl.overall.latency_p95_s": 3.77, - "bfcl.non_live.ast_accuracy": 55.42, - "bfcl.non_live.simple_ast_accuracy": 55.67, - "bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl.non_live.parallel_ast_accuracy": 38.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 44.0, - "bfcl.live.live_accuracy": 52.48, - "bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl.live.live_multiple_ast_accuracy": 49.76, - "bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 16.67, - "bfcl.multi_turn.accuracy": 1.75, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl.web_search.accuracy": 19.5, - "bfcl.web_search.base_accuracy": 20.0, - "bfcl.web_search.no_snippet_accuracy": 19.0, - "bfcl.memory.accuracy": 2.58, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 1.94, - "bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl.relevance.relevance_detection_accuracy": 31.25, - "bfcl.relevance.irrelevance_detection_accuracy": 95.29, - "bfcl.format_sensitivity.max_delta": 67.5, - "bfcl.format_sensitivity.stddev": 20.07 - } - }, - { - "model_id": "anthropic/claude-opus-4-5-20251101-fc", - "name": "Claude-Opus-4-5-20251101 (FC)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 1.0, - "bfcl.overall.overall_accuracy": 77.47, - "bfcl.overall.total_cost_usd": 86.55, - "bfcl.overall.latency_mean_s": 4.38, - "bfcl.overall.latency_std_s": 3.13, - "bfcl.overall.latency_p95_s": 7.56, - "bfcl.non_live.ast_accuracy": 88.58, - "bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 79.79, - "bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl.live.live_multiple_ast_accuracy": 78.16, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 68.38, - "bfcl.multi_turn.base_accuracy": 81.0, - "bfcl.multi_turn.miss_function_accuracy": 64.0, - "bfcl.multi_turn.miss_parameter_accuracy": 58.0, - "bfcl.multi_turn.long_context_accuracy": 70.5, - "bfcl.web_search.accuracy": 84.5, - "bfcl.web_search.base_accuracy": 84.0, - "bfcl.web_search.no_snippet_accuracy": 85.0, - "bfcl.memory.accuracy": 73.76, - "bfcl.memory.kv_accuracy": 70.97, - "bfcl.memory.vector_accuracy": 72.9, - "bfcl.memory.recursive_summarization_accuracy": 77.42, - "bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl.relevance.irrelevance_detection_accuracy": 84.72 - } - }, - { - "model_id": "anthropic/claude-opus-4-5-20251101-prompt", - "name": "Claude-Opus-4-5-20251101 (Prompt)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 57.0, - "bfcl.overall.overall_accuracy": 33.47, - "bfcl.overall.total_cost_usd": 88.33, - "bfcl.overall.latency_mean_s": 3.76, - "bfcl.overall.latency_std_s": 13.19, - "bfcl.overall.latency_p95_s": 5.52, - "bfcl.non_live.ast_accuracy": 89.65, - "bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.5, - "bfcl.live.live_accuracy": 76.02, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 16.12, - "bfcl.multi_turn.base_accuracy": 20.5, - "bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl.web_search.accuracy": 13.0, - "bfcl.web_search.base_accuracy": 13.0, - "bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl.memory.accuracy": 1.94, - "bfcl.memory.kv_accuracy": 1.29, - "bfcl.memory.vector_accuracy": 1.94, - "bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 90.75, - "bfcl.format_sensitivity.max_delta": 13.0, - "bfcl.format_sensitivity.stddev": 3.65 - } - }, - { - "model_id": "anthropic/claude-sonnet-4-5-20250929-fc", - "name": "Claude-Sonnet-4-5-20250929 (FC)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 2.0, - "bfcl.overall.overall_accuracy": 73.24, - "bfcl.overall.total_cost_usd": 43.73, - "bfcl.overall.latency_mean_s": 4.31, - "bfcl.overall.latency_std_s": 4.43, - "bfcl.overall.latency_p95_s": 7.27, - "bfcl.non_live.ast_accuracy": 88.65, - "bfcl.non_live.simple_ast_accuracy": 72.58, - "bfcl.non_live.multiple_ast_accuracy": 95.5, - "bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl.live.live_accuracy": 81.13, - "bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 83.33, - "bfcl.multi_turn.accuracy": 61.37, - "bfcl.multi_turn.base_accuracy": 69.0, - "bfcl.multi_turn.miss_function_accuracy": 65.0, - "bfcl.multi_turn.miss_parameter_accuracy": 52.5, - "bfcl.multi_turn.long_context_accuracy": 59.0, - "bfcl.web_search.accuracy": 81.0, - "bfcl.web_search.base_accuracy": 82.0, - "bfcl.web_search.no_snippet_accuracy": 80.0, - "bfcl.memory.accuracy": 64.95, - "bfcl.memory.kv_accuracy": 54.19, - "bfcl.memory.vector_accuracy": 57.42, - "bfcl.memory.recursive_summarization_accuracy": 83.23, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 86.61 - } - }, - { - "model_id": "anthropic/claude-sonnet-4-5-20250929-prompt", - "name": "Claude-Sonnet-4-5-20250929 (Prompt)", - "developer": "Anthropic", - "scores": { - "bfcl.overall.rank": 89.0, - "bfcl.overall.overall_accuracy": 24.9, - "bfcl.overall.total_cost_usd": 47.82, - "bfcl.overall.latency_mean_s": 3.84, - "bfcl.overall.latency_std_s": 1.53, - "bfcl.overall.latency_p95_s": 6.66, - "bfcl.non_live.ast_accuracy": 59.81, - "bfcl.non_live.simple_ast_accuracy": 47.25, - "bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl.non_live.parallel_ast_accuracy": 53.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 59.0, - "bfcl.live.live_accuracy": 46.56, - "bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl.live.live_multiple_ast_accuracy": 40.17, - "bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl.multi_turn.accuracy": 1.62, - "bfcl.multi_turn.base_accuracy": 2.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl.web_search.accuracy": 16.0, - "bfcl.web_search.base_accuracy": 16.0, - "bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl.memory.accuracy": 5.38, - "bfcl.memory.kv_accuracy": 4.52, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl.relevance.irrelevance_detection_accuracy": 95.03, - "bfcl.format_sensitivity.max_delta": 37.5, - "bfcl.format_sensitivity.stddev": 10.07 - } - }, - { - "model_id": "bittensor/bitagent-bounty-8b", - "name": "BitAgent-Bounty-8B", - "developer": "bittensor", - "scores": { - "bfcl.overall.rank": 36.0, - "bfcl.overall.overall_accuracy": 46.23, - "bfcl.overall.total_cost_usd": 18.02, - "bfcl.overall.latency_mean_s": 16.52, - "bfcl.overall.latency_std_s": 30.73, - "bfcl.overall.latency_p95_s": 77.12, - "bfcl.non_live.ast_accuracy": 81.6, - "bfcl.non_live.simple_ast_accuracy": 72.42, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl.live.live_accuracy": 93.12, - "bfcl.live.live_simple_ast_accuracy": 90.31, - "bfcl.live.live_multiple_ast_accuracy": 94.02, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 95.83, - "bfcl.multi_turn.accuracy": 62.38, - "bfcl.multi_turn.base_accuracy": 75.0, - "bfcl.multi_turn.miss_function_accuracy": 49.5, - "bfcl.multi_turn.miss_parameter_accuracy": 68.0, - "bfcl.multi_turn.long_context_accuracy": 57.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 1.51, - "bfcl.memory.kv_accuracy": 1.29, - "bfcl.memory.vector_accuracy": 1.29, - "bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 97.48 - } - }, - { - "model_id": "cohere/command-a-fc", - "name": "Command A (FC)", - "developer": "cohere", - "scores": { - "bfcl.overall.rank": 35.0, - "bfcl.overall.overall_accuracy": 46.49, - "bfcl.overall.total_cost_usd": 91.37, - "bfcl.overall.latency_mean_s": 2.09, - "bfcl.overall.latency_std_s": 7.36, - "bfcl.overall.latency_p95_s": 4.94, - "bfcl.non_live.ast_accuracy": 87.56, - "bfcl.non_live.simple_ast_accuracy": 75.75, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl.live.live_accuracy": 78.53, - "bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 29.5, - "bfcl.multi_turn.base_accuracy": 38.0, - "bfcl.multi_turn.miss_function_accuracy": 23.0, - "bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl.multi_turn.long_context_accuracy": 25.0, - "bfcl.web_search.accuracy": 46.5, - "bfcl.web_search.base_accuracy": 60.0, - "bfcl.web_search.no_snippet_accuracy": 33.0, - "bfcl.memory.accuracy": 16.56, - "bfcl.memory.kv_accuracy": 4.52, - "bfcl.memory.vector_accuracy": 5.16, - "bfcl.memory.recursive_summarization_accuracy": 40.0, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 84.19 - } - }, - { - "model_id": "cohere/command-a-reasoning-fc", - "name": "Command A Reasoning (FC)", - "developer": "cohere", - "scores": { - "bfcl.overall.rank": 13.0, - "bfcl.overall.overall_accuracy": 57.06, - "bfcl.overall.total_cost_usd": 3.04, - "bfcl.overall.latency_mean_s": 3.44, - "bfcl.overall.latency_std_s": 4.91, - "bfcl.overall.latency_p95_s": 8.39, - "bfcl.non_live.ast_accuracy": 86.27, - "bfcl.non_live.simple_ast_accuracy": 73.58, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 78.61, - "bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl.live.live_multiple_ast_accuracy": 78.35, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 50.12, - "bfcl.multi_turn.base_accuracy": 61.5, - "bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl.multi_turn.miss_parameter_accuracy": 49.5, - "bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl.web_search.accuracy": 55.5, - "bfcl.web_search.base_accuracy": 65.0, - "bfcl.web_search.no_snippet_accuracy": 46.0, - "bfcl.memory.accuracy": 28.82, - "bfcl.memory.kv_accuracy": 16.13, - "bfcl.memory.vector_accuracy": 23.87, - "bfcl.memory.recursive_summarization_accuracy": 46.45, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 86.75 - } - }, - { - "model_id": "cohere/command-r7b-fc", - "name": "Command R7B (FC)", - "developer": "cohere", - "scores": { - "bfcl.overall.rank": 61.0, - "bfcl.overall.overall_accuracy": 32.07, - "bfcl.overall.total_cost_usd": 1.5, - "bfcl.overall.latency_mean_s": 1.38, - "bfcl.overall.latency_std_s": 2.87, - "bfcl.overall.latency_p95_s": 2.69, - "bfcl.non_live.ast_accuracy": 80.96, - "bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl.live.live_accuracy": 69.06, - "bfcl.live.live_simple_ast_accuracy": 62.79, - "bfcl.live.live_multiple_ast_accuracy": 70.94, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 8.25, - "bfcl.multi_turn.base_accuracy": 12.0, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl.multi_turn.long_context_accuracy": 10.0, - "bfcl.web_search.accuracy": 27.0, - "bfcl.web_search.base_accuracy": 43.0, - "bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl.memory.accuracy": 5.16, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 81.65 - } - }, - { - "model_id": "deepseek/deepseek-v3-2-exp-fc", - "name": "DeepSeek-V3.2-Exp (FC)", - "developer": "deepseek", - "scores": { - "bfcl.overall.rank": 19.0, - "bfcl.overall.overall_accuracy": 54.12, - "bfcl.overall.total_cost_usd": 6.71, - "bfcl.overall.latency_mean_s": 5.83, - "bfcl.overall.latency_std_s": 11.71, - "bfcl.overall.latency_p95_s": 10.59, - "bfcl.non_live.ast_accuracy": 34.85, - "bfcl.non_live.simple_ast_accuracy": 37.92, - "bfcl.non_live.multiple_ast_accuracy": 74.0, - "bfcl.non_live.parallel_ast_accuracy": 15.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 12.5, - "bfcl.live.live_accuracy": 53.66, - "bfcl.live.live_simple_ast_accuracy": 66.28, - "bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl.multi_turn.accuracy": 37.38, - "bfcl.multi_turn.base_accuracy": 41.5, - "bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl.multi_turn.long_context_accuracy": 35.0, - "bfcl.web_search.accuracy": 69.5, - "bfcl.web_search.base_accuracy": 80.0, - "bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl.memory.accuracy": 54.19, - "bfcl.memory.kv_accuracy": 41.94, - "bfcl.memory.vector_accuracy": 61.29, - "bfcl.memory.recursive_summarization_accuracy": 59.35, - "bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl.relevance.irrelevance_detection_accuracy": 93.18 - } - }, - { - "model_id": "deepseek/deepseek-v3-2-exp-prompt-thinking", - "name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", - "developer": "deepseek", - "scores": { - "bfcl.overall.rank": 14.0, - "bfcl.overall.overall_accuracy": 56.73, - "bfcl.overall.total_cost_usd": 57.75, - "bfcl.overall.latency_mean_s": 37.89, - "bfcl.overall.latency_std_s": 49.56, - "bfcl.overall.latency_p95_s": 102.09, - "bfcl.non_live.ast_accuracy": 85.52, - "bfcl.non_live.simple_ast_accuracy": 74.08, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 86.5, - "bfcl.live.live_accuracy": 76.02, - "bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl.live.live_multiple_ast_accuracy": 74.74, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 44.88, - "bfcl.multi_turn.base_accuracy": 55.0, - "bfcl.multi_turn.miss_function_accuracy": 49.0, - "bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl.multi_turn.long_context_accuracy": 48.5, - "bfcl.web_search.accuracy": 58.0, - "bfcl.web_search.base_accuracy": 64.0, - "bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl.memory.accuracy": 44.09, - "bfcl.memory.kv_accuracy": 46.45, - "bfcl.memory.vector_accuracy": 46.45, - "bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 67.0, - "bfcl.format_sensitivity.max_delta": 10.0, - "bfcl.format_sensitivity.stddev": 2.77 - } - }, - { - "model_id": "google/gemini-2-5-flash-fc", - "name": "Gemini-2.5-Flash (FC)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 15.0, - "bfcl.overall.overall_accuracy": 56.24, - "bfcl.overall.total_cost_usd": 26.36, - "bfcl.overall.latency_mean_s": 2.99, - "bfcl.overall.latency_std_s": 9.22, - "bfcl.overall.latency_p95_s": 5.62, - "bfcl.non_live.ast_accuracy": 84.96, - "bfcl.non_live.simple_ast_accuracy": 74.33, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 79.5, - "bfcl.live.live_accuracy": 74.39, - "bfcl.live.live_simple_ast_accuracy": 85.27, - "bfcl.live.live_multiple_ast_accuracy": 71.7, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 36.25, - "bfcl.multi_turn.base_accuracy": 41.5, - "bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl.multi_turn.miss_parameter_accuracy": 32.0, - "bfcl.multi_turn.long_context_accuracy": 35.5, - "bfcl.web_search.accuracy": 59.0, - "bfcl.web_search.base_accuracy": 59.0, - "bfcl.web_search.no_snippet_accuracy": 59.0, - "bfcl.memory.accuracy": 41.29, - "bfcl.memory.kv_accuracy": 19.35, - "bfcl.memory.vector_accuracy": 50.32, - "bfcl.memory.recursive_summarization_accuracy": 54.19, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 93.67 - } - }, - { - "model_id": "google/gemini-2-5-flash-lite-fc", - "name": "Gemini-2.5-Flash-Lite (FC)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 52.0, - "bfcl.overall.overall_accuracy": 36.87, - "bfcl.overall.total_cost_usd": 7.55, - "bfcl.overall.latency_mean_s": 1.18, - "bfcl.overall.latency_std_s": 8.06, - "bfcl.overall.latency_p95_s": 1.67, - "bfcl.non_live.ast_accuracy": 86.6, - "bfcl.non_live.simple_ast_accuracy": 70.92, - "bfcl.non_live.multiple_ast_accuracy": 90.0, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl.live.live_accuracy": 65.8, - "bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl.live.live_multiple_ast_accuracy": 63.82, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 13.5, - "bfcl.multi_turn.base_accuracy": 20.0, - "bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl.multi_turn.miss_parameter_accuracy": 15.0, - "bfcl.multi_turn.long_context_accuracy": 17.5, - "bfcl.web_search.accuracy": 21.0, - "bfcl.web_search.base_accuracy": 26.0, - "bfcl.web_search.no_snippet_accuracy": 16.0, - "bfcl.memory.accuracy": 20.65, - "bfcl.memory.kv_accuracy": 3.87, - "bfcl.memory.vector_accuracy": 6.45, - "bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl.relevance.irrelevance_detection_accuracy": 92.5 - } - }, - { - "model_id": "google/gemini-2-5-flash-lite-prompt", - "name": "Gemini-2.5-Flash-Lite (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 73.0, - "bfcl.overall.overall_accuracy": 28.03, - "bfcl.overall.total_cost_usd": 7.05, - "bfcl.overall.latency_mean_s": 1.0, - "bfcl.overall.latency_std_s": 4.75, - "bfcl.overall.latency_p95_s": 1.4, - "bfcl.non_live.ast_accuracy": 83.9, - "bfcl.non_live.simple_ast_accuracy": 70.08, - "bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 54.85, - "bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl.live.live_multiple_ast_accuracy": 51.66, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl.multi_turn.accuracy": 7.63, - "bfcl.multi_turn.base_accuracy": 10.0, - "bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 12.69, - "bfcl.memory.kv_accuracy": 1.94, - "bfcl.memory.vector_accuracy": 6.45, - "bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl.relevance.irrelevance_detection_accuracy": 93.33, - "bfcl.format_sensitivity.max_delta": 25.5, - "bfcl.format_sensitivity.stddev": 6.68 - } - }, - { - "model_id": "google/gemini-2-5-flash-prompt", - "name": "Gemini-2.5-Flash (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 26.0, - "bfcl.overall.overall_accuracy": 50.9, - "bfcl.overall.total_cost_usd": 33.45, - "bfcl.overall.latency_mean_s": 3.18, - "bfcl.overall.latency_std_s": 4.44, - "bfcl.overall.latency_p95_s": 6.09, - "bfcl.non_live.ast_accuracy": 88.08, - "bfcl.non_live.simple_ast_accuracy": 77.33, - "bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl.live.live_accuracy": 78.16, - "bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl.live.live_multiple_ast_accuracy": 75.97, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 16.75, - "bfcl.multi_turn.base_accuracy": 14.5, - "bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl.multi_turn.long_context_accuracy": 18.5, - "bfcl.web_search.accuracy": 62.0, - "bfcl.web_search.base_accuracy": 60.0, - "bfcl.web_search.no_snippet_accuracy": 64.0, - "bfcl.memory.accuracy": 38.71, - "bfcl.memory.kv_accuracy": 13.55, - "bfcl.memory.vector_accuracy": 47.1, - "bfcl.memory.recursive_summarization_accuracy": 55.48, - "bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl.relevance.irrelevance_detection_accuracy": 91.09, - "bfcl.format_sensitivity.max_delta": 9.0, - "bfcl.format_sensitivity.stddev": 2.45 - } - }, - { - "model_id": "google/gemini-3-pro-preview-fc", - "name": "Gemini-3-Pro-Preview (FC)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 7.0, - "bfcl.overall.overall_accuracy": 68.14, - "bfcl.overall.total_cost_usd": 224.69, - "bfcl.overall.latency_mean_s": 15.87, - "bfcl.overall.latency_std_s": 41.41, - "bfcl.overall.latency_p95_s": 58.48, - "bfcl.non_live.ast_accuracy": 85.75, - "bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl.live.live_accuracy": 81.72, - "bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl.live.live_multiple_ast_accuracy": 80.44, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 63.12, - "bfcl.multi_turn.base_accuracy": 69.0, - "bfcl.multi_turn.miss_function_accuracy": 63.0, - "bfcl.multi_turn.miss_parameter_accuracy": 56.5, - "bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl.web_search.accuracy": 68.5, - "bfcl.web_search.base_accuracy": 63.0, - "bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl.memory.accuracy": 54.84, - "bfcl.memory.kv_accuracy": 50.32, - "bfcl.memory.vector_accuracy": 63.23, - "bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 77.85 - } - }, - { - "model_id": "google/gemini-3-pro-preview-prompt", - "name": "Gemini-3-Pro-Preview (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 3.0, - "bfcl.overall.overall_accuracy": 72.51, - "bfcl.overall.total_cost_usd": 298.47, - "bfcl.overall.latency_mean_s": 12.08, - "bfcl.overall.latency_std_s": 21.3, - "bfcl.overall.latency_p95_s": 32.73, - "bfcl.non_live.ast_accuracy": 90.65, - "bfcl.non_live.simple_ast_accuracy": 79.58, - "bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl.non_live.parallel_ast_accuracy": 95.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl.live.live_accuracy": 83.12, - "bfcl.live.live_simple_ast_accuracy": 87.6, - "bfcl.live.live_multiple_ast_accuracy": 81.77, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 87.5, - "bfcl.multi_turn.accuracy": 60.75, - "bfcl.multi_turn.base_accuracy": 64.5, - "bfcl.multi_turn.miss_function_accuracy": 60.0, - "bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl.multi_turn.long_context_accuracy": 64.0, - "bfcl.web_search.accuracy": 80.0, - "bfcl.web_search.base_accuracy": 78.0, - "bfcl.web_search.no_snippet_accuracy": 82.0, - "bfcl.memory.accuracy": 61.72, - "bfcl.memory.kv_accuracy": 59.35, - "bfcl.memory.vector_accuracy": 62.58, - "bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 85.59, - "bfcl.format_sensitivity.max_delta": 8.5, - "bfcl.format_sensitivity.stddev": 1.7 - } - }, - { - "model_id": "google/gemma-3-12b-it-prompt", - "name": "Gemma-3-12b-it (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 66.0, - "bfcl.overall.overall_accuracy": 30.43, - "bfcl.overall.total_cost_usd": 10.77, - "bfcl.overall.latency_mean_s": 11.1, - "bfcl.overall.latency_std_s": 17.17, - "bfcl.overall.latency_p95_s": 34.66, - "bfcl.non_live.ast_accuracy": 79.44, - "bfcl.non_live.simple_ast_accuracy": 76.25, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 56.5, - "bfcl.live.live_accuracy": 74.24, - "bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl.live.live_multiple_ast_accuracy": 71.89, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl.multi_turn.accuracy": 5.75, - "bfcl.multi_turn.base_accuracy": 6.5, - "bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl.web_search.accuracy": 4.0, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl.memory.accuracy": 27.53, - "bfcl.memory.kv_accuracy": 8.39, - "bfcl.memory.vector_accuracy": 25.16, - "bfcl.memory.recursive_summarization_accuracy": 49.03, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 70.29, - "bfcl.format_sensitivity.max_delta": 67.5, - "bfcl.format_sensitivity.stddev": 22.41 - } - }, - { - "model_id": "google/gemma-3-1b-it-prompt", - "name": "Gemma-3-1b-it (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 109.0, - "bfcl.overall.overall_accuracy": 7.17, - "bfcl.overall.total_cost_usd": 3.4, - "bfcl.overall.latency_mean_s": 3.98, - "bfcl.overall.latency_std_s": 9.8, - "bfcl.overall.latency_p95_s": 12.06, - "bfcl.non_live.ast_accuracy": 20.21, - "bfcl.non_live.simple_ast_accuracy": 43.33, - "bfcl.non_live.multiple_ast_accuracy": 36.0, - "bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 1.5, - "bfcl.live.live_accuracy": 11.84, - "bfcl.live.live_simple_ast_accuracy": 36.43, - "bfcl.live.live_multiple_ast_accuracy": 6.27, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 0.0, - "bfcl.multi_turn.base_accuracy": 0.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 3.23, - "bfcl.memory.kv_accuracy": 3.87, - "bfcl.memory.vector_accuracy": 3.87, - "bfcl.memory.recursive_summarization_accuracy": 1.94, - "bfcl.relevance.relevance_detection_accuracy": 37.5, - "bfcl.relevance.irrelevance_detection_accuracy": 33.18, - "bfcl.format_sensitivity.max_delta": 25.5, - "bfcl.format_sensitivity.stddev": 9.76 - } - }, - { - "model_id": "google/gemma-3-27b-it-prompt", - "name": "Gemma-3-27b-it (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 69.0, - "bfcl.overall.overall_accuracy": 29.47, - "bfcl.overall.total_cost_usd": 11.82, - "bfcl.overall.latency_mean_s": 10.88, - "bfcl.overall.latency_std_s": 19.67, - "bfcl.overall.latency_p95_s": 55.5, - "bfcl.non_live.ast_accuracy": 87.17, - "bfcl.non_live.simple_ast_accuracy": 77.67, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 74.54, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 72.46, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl.multi_turn.accuracy": 10.75, - "bfcl.multi_turn.base_accuracy": 16.5, - "bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl.multi_turn.long_context_accuracy": 14.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 13.55, - "bfcl.memory.kv_accuracy": 1.94, - "bfcl.memory.vector_accuracy": 3.23, - "bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 73.67, - "bfcl.format_sensitivity.max_delta": 34.0, - "bfcl.format_sensitivity.stddev": 8.06 - } - }, - { - "model_id": "google/gemma-3-4b-it-prompt", - "name": "Gemma-3-4b-it (Prompt)", - "developer": "Google", - "scores": { - "bfcl.overall.rank": 101.0, - "bfcl.overall.overall_accuracy": 19.62, - "bfcl.overall.total_cost_usd": 4.14, - "bfcl.overall.latency_mean_s": 4.69, - "bfcl.overall.latency_std_s": 9.53, - "bfcl.overall.latency_p95_s": 11.42, - "bfcl.non_live.ast_accuracy": 61.12, - "bfcl.non_live.simple_ast_accuracy": 64.5, - "bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl.non_live.parallel_ast_accuracy": 56.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 36.0, - "bfcl.live.live_accuracy": 60.84, - "bfcl.live.live_simple_ast_accuracy": 70.93, - "bfcl.live.live_multiple_ast_accuracy": 59.35, - "bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl.multi_turn.accuracy": 0.38, - "bfcl.multi_turn.base_accuracy": 0.5, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl.web_search.accuracy": 1.0, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 8.6, - "bfcl.memory.kv_accuracy": 9.68, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 53.94, - "bfcl.format_sensitivity.max_delta": 69.5, - "bfcl.format_sensitivity.stddev": 23.67 - } - }, - { - "model_id": "huawei-noah-ustc/toolace-2-8b-fc", - "name": "ToolACE-2-8B (FC)", - "developer": "huawei-noah-ustc", - "scores": { - "bfcl.overall.rank": 40.0, - "bfcl.overall.overall_accuracy": 42.44, - "bfcl.overall.total_cost_usd": 24.43, - "bfcl.overall.latency_mean_s": 15.95, - "bfcl.overall.latency_std_s": 40.06, - "bfcl.overall.latency_p95_s": 65.26, - "bfcl.non_live.ast_accuracy": 87.1, - "bfcl.non_live.simple_ast_accuracy": 73.42, - "bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl.live.live_accuracy": 77.42, - "bfcl.live.live_simple_ast_accuracy": 71.32, - "bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 38.38, - "bfcl.multi_turn.base_accuracy": 49.0, - "bfcl.multi_turn.miss_function_accuracy": 28.0, - "bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl.multi_turn.long_context_accuracy": 46.0, - "bfcl.web_search.accuracy": 8.5, - "bfcl.web_search.base_accuracy": 13.0, - "bfcl.web_search.no_snippet_accuracy": 4.0, - "bfcl.memory.accuracy": 18.49, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 16.13, - "bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 90.79, - "bfcl.format_sensitivity.max_delta": 81.5, - "bfcl.format_sensitivity.stddev": 27.92 - } - }, - { - "model_id": "ibm/granite-20b-functioncalling-fc", - "name": "Granite-20b-FunctionCalling (FC)", - "developer": "ibm", - "scores": { - "bfcl.overall.rank": 93.0, - "bfcl.overall.overall_accuracy": 23.23, - "bfcl.overall.total_cost_usd": 5.23, - "bfcl.overall.latency_mean_s": 3.2, - "bfcl.overall.latency_std_s": 3.43, - "bfcl.overall.latency_p95_s": 9.97, - "bfcl.non_live.ast_accuracy": 82.35, - "bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl.non_live.multiple_ast_accuracy": 91.5, - "bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl.live.live_accuracy": 58.7, - "bfcl.live.live_simple_ast_accuracy": 67.83, - "bfcl.live.live_multiple_ast_accuracy": 56.7, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 5.38, - "bfcl.multi_turn.base_accuracy": 9.0, - "bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl.multi_turn.miss_parameter_accuracy": 6.5, - "bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 0.0, - "bfcl.memory.kv_accuracy": 0.0, - "bfcl.memory.vector_accuracy": 0.0, - "bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 75.13 - } - }, - { - "model_id": "ibm/granite-3-1-8b-instruct-fc", - "name": "Granite-3.1-8B-Instruct (FC)", - "developer": "ibm", - "scores": { - "bfcl.overall.rank": 81.0, - "bfcl.overall.overall_accuracy": 27.1, - "bfcl.overall.total_cost_usd": 9.32, - "bfcl.overall.latency_mean_s": 13.23, - "bfcl.overall.latency_std_s": 31.28, - "bfcl.overall.latency_p95_s": 65.19, - "bfcl.non_live.ast_accuracy": 78.33, - "bfcl.non_live.simple_ast_accuracy": 67.33, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 70.0, - "bfcl.live.live_accuracy": 60.33, - "bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl.live.live_multiple_ast_accuracy": 61.82, - "bfcl.live.live_parallel_ast_accuracy": 18.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl.multi_turn.accuracy": 7.5, - "bfcl.multi_turn.base_accuracy": 11.5, - "bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 14.41, - "bfcl.memory.kv_accuracy": 9.68, - "bfcl.memory.vector_accuracy": 7.1, - "bfcl.memory.recursive_summarization_accuracy": 26.45, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 79.98 - } - }, - { - "model_id": "ibm/granite-3-2-8b-instruct-fc", - "name": "Granite-3.2-8B-Instruct (FC)", - "developer": "ibm", - "scores": { - "bfcl.overall.rank": 83.0, - "bfcl.overall.overall_accuracy": 26.87, - "bfcl.overall.total_cost_usd": 25.02, - "bfcl.overall.latency_mean_s": 36.13, - "bfcl.overall.latency_std_s": 81.76, - "bfcl.overall.latency_p95_s": 216.28, - "bfcl.non_live.ast_accuracy": 79.77, - "bfcl.non_live.simple_ast_accuracy": 69.58, - "bfcl.non_live.multiple_ast_accuracy": 88.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl.live.live_accuracy": 60.33, - "bfcl.live.live_simple_ast_accuracy": 60.47, - "bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl.multi_turn.accuracy": 7.38, - "bfcl.multi_turn.base_accuracy": 9.5, - "bfcl.multi_turn.miss_function_accuracy": 3.0, - "bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl.multi_turn.long_context_accuracy": 9.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 12.47, - "bfcl.memory.kv_accuracy": 6.45, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 80.53 - } - }, - { - "model_id": "ibm/granite-4-0-350m-fc", - "name": "Granite-4.0-350m (FC)", - "developer": "ibm", - "scores": { - "bfcl.overall.rank": 103.0, - "bfcl.overall.overall_accuracy": 18.98, - "bfcl.overall.total_cost_usd": 1.44, - "bfcl.overall.latency_mean_s": 1.74, - "bfcl.overall.latency_std_s": 4.85, - "bfcl.overall.latency_p95_s": 3.44, - "bfcl.non_live.ast_accuracy": 67.92, - "bfcl.non_live.simple_ast_accuracy": 61.67, - "bfcl.non_live.multiple_ast_accuracy": 84.5, - "bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 55.5, - "bfcl.live.live_accuracy": 46.11, - "bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl.live.live_multiple_ast_accuracy": 42.36, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl.multi_turn.accuracy": 2.5, - "bfcl.multi_turn.base_accuracy": 5.0, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 3.23, - "bfcl.memory.kv_accuracy": 1.94, - "bfcl.memory.vector_accuracy": 1.29, - "bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 60.84 - } - }, - { - "model_id": "katanemo/arch-agent-1-5b", - "name": "Arch-Agent-1.5B", - "developer": "katanemo", - "scores": { - "bfcl.overall.rank": 60.0, - "bfcl.overall.overall_accuracy": 32.14, - "bfcl.overall.total_cost_usd": 2.45, - "bfcl.overall.latency_mean_s": 2.38, - "bfcl.overall.latency_std_s": 4.01, - "bfcl.overall.latency_p95_s": 5.3, - "bfcl.non_live.ast_accuracy": 82.67, - "bfcl.non_live.simple_ast_accuracy": 72.17, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 81.0, - "bfcl.live.live_accuracy": 67.73, - "bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl.live.live_parallel_ast_accuracy": 31.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 26.62, - "bfcl.multi_turn.base_accuracy": 35.5, - "bfcl.multi_turn.miss_function_accuracy": 27.5, - "bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl.multi_turn.long_context_accuracy": 22.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 8.17, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 5.81, - "bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 74.83 - } - }, - { - "model_id": "katanemo/arch-agent-32b", - "name": "Arch-Agent-32B", - "developer": "katanemo", - "scores": { - "bfcl.overall.rank": 37.0, - "bfcl.overall.overall_accuracy": 45.37, - "bfcl.overall.total_cost_usd": 8.87, - "bfcl.overall.latency_mean_s": 9.44, - "bfcl.overall.latency_std_s": 21.44, - "bfcl.overall.latency_p95_s": 24.87, - "bfcl.non_live.ast_accuracy": 88.92, - "bfcl.non_live.simple_ast_accuracy": 76.67, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 96.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl.live.live_accuracy": 80.68, - "bfcl.live.live_simple_ast_accuracy": 86.43, - "bfcl.live.live_multiple_ast_accuracy": 79.11, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 54.25, - "bfcl.multi_turn.base_accuracy": 64.5, - "bfcl.multi_turn.miss_function_accuracy": 58.0, - "bfcl.multi_turn.miss_parameter_accuracy": 53.0, - "bfcl.multi_turn.long_context_accuracy": 41.5, - "bfcl.web_search.accuracy": 5.0, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl.memory.accuracy": 14.62, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 9.03, - "bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 82.15 - } - }, - { - "model_id": "katanemo/arch-agent-3b", - "name": "Arch-Agent-3B", - "developer": "katanemo", - "scores": { - "bfcl.overall.rank": 56.0, - "bfcl.overall.overall_accuracy": 35.36, - "bfcl.overall.total_cost_usd": 3.7, - "bfcl.overall.latency_mean_s": 3.56, - "bfcl.overall.latency_std_s": 6.65, - "bfcl.overall.latency_p95_s": 8.19, - "bfcl.non_live.ast_accuracy": 86.67, - "bfcl.non_live.simple_ast_accuracy": 78.67, - "bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl.live.live_accuracy": 72.91, - "bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl.live.live_multiple_ast_accuracy": 72.27, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 34.88, - "bfcl.multi_turn.base_accuracy": 42.0, - "bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 6.88, - "bfcl.memory.kv_accuracy": 5.16, - "bfcl.memory.vector_accuracy": 5.81, - "bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 74.67 - } - }, - { - "model_id": "madeagents/hammer2-1-0-5b-fc", - "name": "Hammer2.1-0.5b (FC)", - "developer": "madeagents", - "scores": { - "bfcl.overall.rank": 100.0, - "bfcl.overall.overall_accuracy": 21.22, - "bfcl.overall.total_cost_usd": 2.82, - "bfcl.overall.latency_mean_s": 2.79, - "bfcl.overall.latency_std_s": 3.17, - "bfcl.overall.latency_p95_s": 9.86, - "bfcl.non_live.ast_accuracy": 65.98, - "bfcl.non_live.simple_ast_accuracy": 62.42, - "bfcl.non_live.multiple_ast_accuracy": 81.0, - "bfcl.non_live.parallel_ast_accuracy": 69.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 51.5, - "bfcl.live.live_accuracy": 54.63, - "bfcl.live.live_simple_ast_accuracy": 56.59, - "bfcl.live.live_multiple_ast_accuracy": 54.42, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 2.88, - "bfcl.multi_turn.base_accuracy": 4.5, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 1.08, - "bfcl.memory.kv_accuracy": 0.65, - "bfcl.memory.vector_accuracy": 1.94, - "bfcl.memory.recursive_summarization_accuracy": 0.65, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 80.79 - } - }, - { - "model_id": "madeagents/hammer2-1-1-5b-fc", - "name": "Hammer2.1-1.5b (FC)", - "developer": "madeagents", - "scores": { - "bfcl.overall.rank": 75.0, - "bfcl.overall.overall_accuracy": 27.88, - "bfcl.overall.total_cost_usd": 6.83, - "bfcl.overall.latency_mean_s": 6.28, - "bfcl.overall.latency_std_s": 8.79, - "bfcl.overall.latency_p95_s": 30.72, - "bfcl.non_live.ast_accuracy": 82.98, - "bfcl.non_live.simple_ast_accuracy": 73.92, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl.live.live_accuracy": 69.5, - "bfcl.live.live_simple_ast_accuracy": 72.09, - "bfcl.live.live_multiple_ast_accuracy": 69.33, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 15.62, - "bfcl.multi_turn.base_accuracy": 20.0, - "bfcl.multi_turn.miss_function_accuracy": 16.5, - "bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 0.0, - "bfcl.memory.kv_accuracy": 0.0, - "bfcl.memory.vector_accuracy": 0.0, - "bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 79.4 - } - }, - { - "model_id": "madeagents/hammer2-1-3b-fc", - "name": "Hammer2.1-3b (FC)", - "developer": "madeagents", - "scores": { - "bfcl.overall.rank": 68.0, - "bfcl.overall.overall_accuracy": 29.71, - "bfcl.overall.total_cost_usd": 10.89, - "bfcl.overall.latency_mean_s": 11.24, - "bfcl.overall.latency_std_s": 15.81, - "bfcl.overall.latency_p95_s": 47.44, - "bfcl.non_live.ast_accuracy": 84.96, - "bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl.live.live_accuracy": 70.54, - "bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl.live.live_multiple_ast_accuracy": 71.32, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 16.5, - "bfcl.multi_turn.base_accuracy": 22.0, - "bfcl.multi_turn.miss_function_accuracy": 12.5, - "bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl.multi_turn.long_context_accuracy": 15.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 3.01, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 3.87, - "bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl.relevance.irrelevance_detection_accuracy": 86.12 - } - }, - { - "model_id": "madeagents/hammer2-1-7b-fc", - "name": "Hammer2.1-7b (FC)", - "developer": "madeagents", - "scores": { - "bfcl.overall.rank": 64.0, - "bfcl.overall.overall_accuracy": 31.67, - "bfcl.overall.total_cost_usd": 4.99, - "bfcl.overall.latency_mean_s": 5.77, - "bfcl.overall.latency_std_s": 10.29, - "bfcl.overall.latency_p95_s": 31.26, - "bfcl.non_live.ast_accuracy": 85.5, - "bfcl.non_live.simple_ast_accuracy": 72.5, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 86.0, - "bfcl.live.live_accuracy": 69.5, - "bfcl.live.live_simple_ast_accuracy": 66.67, - "bfcl.live.live_multiple_ast_accuracy": 69.99, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 23.87, - "bfcl.multi_turn.base_accuracy": 24.0, - "bfcl.multi_turn.miss_function_accuracy": 28.5, - "bfcl.multi_turn.miss_parameter_accuracy": 21.5, - "bfcl.multi_turn.long_context_accuracy": 21.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 0.0, - "bfcl.memory.kv_accuracy": 0.0, - "bfcl.memory.vector_accuracy": 0.0, - "bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl.relevance.irrelevance_detection_accuracy": 90.12 - } - }, - { - "model_id": "meta/llama-3-1-8b-instruct-prompt", - "name": "Llama-3.1-8B-Instruct (Prompt)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 85.0, - "bfcl.overall.overall_accuracy": 25.83, - "bfcl.overall.total_cost_usd": 7.49, - "bfcl.overall.latency_mean_s": 5.6, - "bfcl.overall.latency_std_s": 19.37, - "bfcl.overall.latency_p95_s": 22.6, - "bfcl.non_live.ast_accuracy": 84.0, - "bfcl.non_live.simple_ast_accuracy": 71.0, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 82.5, - "bfcl.live.live_accuracy": 70.76, - "bfcl.live.live_simple_ast_accuracy": 72.87, - "bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl.multi_turn.accuracy": 11.12, - "bfcl.multi_turn.base_accuracy": 13.0, - "bfcl.multi_turn.miss_function_accuracy": 9.0, - "bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl.web_search.accuracy": 3.0, - "bfcl.web_search.base_accuracy": 6.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 10.75, - "bfcl.memory.kv_accuracy": 7.74, - "bfcl.memory.vector_accuracy": 5.81, - "bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 42.7, - "bfcl.format_sensitivity.max_delta": 74.5, - "bfcl.format_sensitivity.stddev": 29.1 - } - }, - { - "model_id": "meta/llama-3-2-1b-instruct-fc", - "name": "Llama-3.2-1B-Instruct (FC)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 107.0, - "bfcl.overall.overall_accuracy": 10.82, - "bfcl.overall.total_cost_usd": 1.64, - "bfcl.overall.latency_mean_s": 3.21, - "bfcl.overall.latency_std_s": 10.04, - "bfcl.overall.latency_p95_s": 9.77, - "bfcl.non_live.ast_accuracy": 38.38, - "bfcl.non_live.simple_ast_accuracy": 44.0, - "bfcl.non_live.multiple_ast_accuracy": 50.5, - "bfcl.non_live.parallel_ast_accuracy": 44.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 15.0, - "bfcl.live.live_accuracy": 11.77, - "bfcl.live.live_simple_ast_accuracy": 31.78, - "bfcl.live.live_multiple_ast_accuracy": 7.31, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 0.0, - "bfcl.multi_turn.base_accuracy": 0.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 3.23, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 2.58, - "bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl.relevance.relevance_detection_accuracy": 43.75, - "bfcl.relevance.irrelevance_detection_accuracy": 51.57 - } - }, - { - "model_id": "meta/llama-3-2-3b-instruct-fc", - "name": "Llama-3.2-3B-Instruct (FC)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 98.0, - "bfcl.overall.overall_accuracy": 21.95, - "bfcl.overall.total_cost_usd": 6.2, - "bfcl.overall.latency_mean_s": 6.1, - "bfcl.overall.latency_std_s": 20.07, - "bfcl.overall.latency_p95_s": 17.27, - "bfcl.non_live.ast_accuracy": 82.67, - "bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl.live.live_accuracy": 58.33, - "bfcl.live.live_simple_ast_accuracy": 65.12, - "bfcl.live.live_multiple_ast_accuracy": 57.64, - "bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 4.0, - "bfcl.multi_turn.base_accuracy": 5.0, - "bfcl.multi_turn.miss_function_accuracy": 3.5, - "bfcl.multi_turn.miss_parameter_accuracy": 4.0, - "bfcl.multi_turn.long_context_accuracy": 3.5, - "bfcl.web_search.accuracy": 1.0, - "bfcl.web_search.base_accuracy": 2.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 6.24, - "bfcl.memory.kv_accuracy": 3.23, - "bfcl.memory.vector_accuracy": 3.23, - "bfcl.memory.recursive_summarization_accuracy": 12.26, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 52.06 - } - }, - { - "model_id": "meta/llama-3-3-70b-instruct-fc", - "name": "Llama-3.3-70B-Instruct (FC)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 62.0, - "bfcl.overall.overall_accuracy": 31.9, - "bfcl.overall.total_cost_usd": 29.54, - "bfcl.overall.latency_mean_s": 26.11, - "bfcl.overall.latency_std_s": 93.22, - "bfcl.overall.latency_p95_s": 187.93, - "bfcl.non_live.ast_accuracy": 88.02, - "bfcl.non_live.simple_ast_accuracy": 76.08, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 90.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl.live.live_accuracy": 76.61, - "bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl.live.live_multiple_ast_accuracy": 75.5, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 21.5, - "bfcl.multi_turn.base_accuracy": 26.0, - "bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl.web_search.accuracy": 10.0, - "bfcl.web_search.base_accuracy": 14.0, - "bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl.memory.accuracy": 8.17, - "bfcl.memory.kv_accuracy": 4.52, - "bfcl.memory.vector_accuracy": 8.39, - "bfcl.memory.recursive_summarization_accuracy": 11.61, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 53.53 - } - }, - { - "model_id": "meta/llama-4-maverick-17b-128e-instruct-fp8-fc", - "name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 50.0, - "bfcl.overall.overall_accuracy": 37.29, - "bfcl.overall.total_cost_usd": 18.25, - "bfcl.overall.latency_mean_s": 18.43, - "bfcl.overall.latency_std_s": 34.11, - "bfcl.overall.latency_p95_s": 102.75, - "bfcl.non_live.ast_accuracy": 88.65, - "bfcl.non_live.simple_ast_accuracy": 77.08, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 73.65, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 71.04, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 20.25, - "bfcl.multi_turn.base_accuracy": 27.0, - "bfcl.multi_turn.miss_function_accuracy": 22.0, - "bfcl.multi_turn.miss_parameter_accuracy": 14.0, - "bfcl.multi_turn.long_context_accuracy": 18.0, - "bfcl.web_search.accuracy": 28.0, - "bfcl.web_search.base_accuracy": 39.0, - "bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl.memory.accuracy": 18.92, - "bfcl.memory.kv_accuracy": 8.39, - "bfcl.memory.vector_accuracy": 32.9, - "bfcl.memory.recursive_summarization_accuracy": 15.48, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 55.97 - } - }, - { - "model_id": "meta/llama-4-scout-17b-16e-instruct-fc", - "name": "Llama-4-Scout-17B-16E-Instruct (FC)", - "developer": "Meta", - "scores": { - "bfcl.overall.rank": 72.0, - "bfcl.overall.overall_accuracy": 28.13, - "bfcl.overall.total_cost_usd": 24.68, - "bfcl.overall.latency_mean_s": 17.86, - "bfcl.overall.latency_std_s": 50.68, - "bfcl.overall.latency_p95_s": 166.2, - "bfcl.non_live.ast_accuracy": 89.38, - "bfcl.non_live.simple_ast_accuracy": 79.0, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl.live.live_accuracy": 74.69, - "bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl.live.live_multiple_ast_accuracy": 72.74, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 9.0, - "bfcl.multi_turn.base_accuracy": 12.0, - "bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl.web_search.accuracy": 14.5, - "bfcl.web_search.base_accuracy": 18.0, - "bfcl.web_search.no_snippet_accuracy": 11.0, - "bfcl.memory.accuracy": 8.17, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 2.58, - "bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 44.92 - } - }, - { - "model_id": "microsoft/phi-4-prompt", - "name": "Phi-4 (Prompt)", - "developer": "microsoft", - "scores": { - "bfcl.overall.rank": 70.0, - "bfcl.overall.overall_accuracy": 28.79, - "bfcl.overall.total_cost_usd": 8.72, - "bfcl.overall.latency_mean_s": 9.49, - "bfcl.overall.latency_std_s": 26.73, - "bfcl.overall.latency_p95_s": 23.02, - "bfcl.non_live.ast_accuracy": 69.56, - "bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl.non_live.parallel_ast_accuracy": 65.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 49.5, - "bfcl.live.live_accuracy": 60.7, - "bfcl.live.live_simple_ast_accuracy": 65.5, - "bfcl.live.live_multiple_ast_accuracy": 59.64, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl.multi_turn.accuracy": 3.88, - "bfcl.multi_turn.base_accuracy": 9.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 3.5, - "bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl.web_search.accuracy": 4.5, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl.memory.accuracy": 24.73, - "bfcl.memory.kv_accuracy": 17.42, - "bfcl.memory.vector_accuracy": 25.16, - "bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl.relevance.relevance_detection_accuracy": 50.0, - "bfcl.relevance.irrelevance_detection_accuracy": 87.55, - "bfcl.format_sensitivity.max_delta": 81.5, - "bfcl.format_sensitivity.stddev": 23.34 - } - }, - { - "model_id": "mistralai/ministral-8b-instruct-2410-fc", - "name": "Ministral-8B-Instruct-2410 (FC)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 105.0, - "bfcl.overall.overall_accuracy": 11.1, - "bfcl.overall.total_cost_usd": 70.01, - "bfcl.overall.latency_mean_s": 82.07, - "bfcl.overall.latency_std_s": 212.99, - "bfcl.overall.latency_p95_s": 568.59, - "bfcl.non_live.ast_accuracy": 0.0, - "bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl.live.live_accuracy": 0.0, - "bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 0.0, - "bfcl.multi_turn.base_accuracy": 0.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl.web_search.accuracy": 1.0, - "bfcl.web_search.base_accuracy": 2.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 4.52, - "bfcl.memory.kv_accuracy": 3.87, - "bfcl.memory.vector_accuracy": 7.1, - "bfcl.memory.recursive_summarization_accuracy": 2.58, - "bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl.relevance.irrelevance_detection_accuracy": 100.0, - "bfcl.format_sensitivity.max_delta": 0.0, - "bfcl.format_sensitivity.stddev": 0.0 - } - }, - { - "model_id": "mistralai/mistral-large-2411-fc", - "name": "mistral-large-2411 (FC)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 46.0, - "bfcl.overall.overall_accuracy": 38.37, - "bfcl.overall.total_cost_usd": 115.98, - "bfcl.overall.latency_mean_s": 2.04, - "bfcl.overall.latency_std_s": 4.02, - "bfcl.overall.latency_p95_s": 4.68, - "bfcl.non_live.ast_accuracy": 84.65, - "bfcl.non_live.simple_ast_accuracy": 72.08, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 89.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl.live.live_accuracy": 81.87, - "bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl.live.live_multiple_ast_accuracy": 80.72, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 14.12, - "bfcl.multi_turn.base_accuracy": 18.5, - "bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl.multi_turn.miss_parameter_accuracy": 13.0, - "bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl.web_search.accuracy": 28.0, - "bfcl.web_search.base_accuracy": 41.0, - "bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl.memory.accuracy": 24.95, - "bfcl.memory.kv_accuracy": 18.71, - "bfcl.memory.vector_accuracy": 29.03, - "bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 68.92 - } - }, - { - "model_id": "mistralai/mistral-large-2411-prompt", - "name": "mistral-large-2411 (Prompt)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 63.0, - "bfcl.overall.overall_accuracy": 31.84, - "bfcl.overall.total_cost_usd": 232.42, - "bfcl.overall.latency_mean_s": 1.82, - "bfcl.overall.latency_std_s": 7.15, - "bfcl.overall.latency_p95_s": 4.08, - "bfcl.non_live.ast_accuracy": 83.0, - "bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl.non_live.multiple_ast_accuracy": 89.5, - "bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl.live.live_accuracy": 68.1, - "bfcl.live.live_simple_ast_accuracy": 83.72, - "bfcl.live.live_multiple_ast_accuracy": 64.01, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 13.75, - "bfcl.multi_turn.base_accuracy": 20.0, - "bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl.multi_turn.miss_parameter_accuracy": 11.0, - "bfcl.multi_turn.long_context_accuracy": 19.0, - "bfcl.web_search.accuracy": 20.0, - "bfcl.web_search.base_accuracy": 28.0, - "bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl.memory.accuracy": 23.66, - "bfcl.memory.kv_accuracy": 16.77, - "bfcl.memory.vector_accuracy": 30.97, - "bfcl.memory.recursive_summarization_accuracy": 23.23, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 38.77, - "bfcl.format_sensitivity.max_delta": 13.5, - "bfcl.format_sensitivity.stddev": 3.91 - } - }, - { - "model_id": "mistralai/mistral-medium-2505", - "name": "Mistral-Medium-2505", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 48.0, - "bfcl.overall.overall_accuracy": 37.69, - "bfcl.overall.total_cost_usd": 36.51, - "bfcl.overall.latency_mean_s": 1.21, - "bfcl.overall.latency_std_s": 3.5, - "bfcl.overall.latency_p95_s": 2.86, - "bfcl.non_live.ast_accuracy": 85.33, - "bfcl.non_live.simple_ast_accuracy": 76.33, - "bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 85.5, - "bfcl.live.live_accuracy": 66.03, - "bfcl.live.live_simple_ast_accuracy": 80.23, - "bfcl.live.live_multiple_ast_accuracy": 62.39, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 9.88, - "bfcl.multi_turn.base_accuracy": 13.5, - "bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl.multi_turn.miss_parameter_accuracy": 6.0, - "bfcl.multi_turn.long_context_accuracy": 13.5, - "bfcl.web_search.accuracy": 39.0, - "bfcl.web_search.base_accuracy": 41.0, - "bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl.memory.accuracy": 21.72, - "bfcl.memory.kv_accuracy": 16.13, - "bfcl.memory.vector_accuracy": 14.84, - "bfcl.memory.recursive_summarization_accuracy": 34.19, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 74.49, - "bfcl.format_sensitivity.max_delta": 21.5, - "bfcl.format_sensitivity.stddev": 5.02 - } - }, - { - "model_id": "mistralai/mistral-medium-2505-fc", - "name": "Mistral-Medium-2505 (FC)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 49.0, - "bfcl.overall.overall_accuracy": 37.56, - "bfcl.overall.total_cost_usd": 18.8, - "bfcl.overall.latency_mean_s": 1.6, - "bfcl.overall.latency_std_s": 4.44, - "bfcl.overall.latency_p95_s": 4.19, - "bfcl.non_live.ast_accuracy": 67.44, - "bfcl.non_live.simple_ast_accuracy": 39.75, - "bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl.non_live.parallel_ast_accuracy": 83.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl.live.live_accuracy": 67.95, - "bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl.live.live_multiple_ast_accuracy": 68.09, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 10.75, - "bfcl.multi_turn.base_accuracy": 15.5, - "bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl.multi_turn.long_context_accuracy": 13.0, - "bfcl.web_search.accuracy": 35.0, - "bfcl.web_search.base_accuracy": 36.0, - "bfcl.web_search.no_snippet_accuracy": 34.0, - "bfcl.memory.accuracy": 23.01, - "bfcl.memory.kv_accuracy": 15.48, - "bfcl.memory.vector_accuracy": 20.0, - "bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl.relevance.irrelevance_detection_accuracy": 91.95 - } - }, - { - "model_id": "mistralai/mistral-small-2506-fc", - "name": "Mistral-small-2506 (FC)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 51.0, - "bfcl.overall.overall_accuracy": 37.15, - "bfcl.overall.total_cost_usd": 5.2, - "bfcl.overall.latency_mean_s": 1.48, - "bfcl.overall.latency_std_s": 18.25, - "bfcl.overall.latency_p95_s": 2.5, - "bfcl.non_live.ast_accuracy": 73.6, - "bfcl.non_live.simple_ast_accuracy": 38.92, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 83.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 78.5, - "bfcl.live.live_accuracy": 77.28, - "bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl.live.live_multiple_ast_accuracy": 79.39, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 11.5, - "bfcl.multi_turn.base_accuracy": 17.5, - "bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl.multi_turn.miss_parameter_accuracy": 10.5, - "bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl.web_search.accuracy": 31.0, - "bfcl.web_search.base_accuracy": 37.0, - "bfcl.web_search.no_snippet_accuracy": 25.0, - "bfcl.memory.accuracy": 18.06, - "bfcl.memory.kv_accuracy": 8.39, - "bfcl.memory.vector_accuracy": 14.19, - "bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 87.94 - } - }, - { - "model_id": "mistralai/mistral-small-2506-prompt", - "name": "Mistral-Small-2506 (Prompt)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 59.0, - "bfcl.overall.overall_accuracy": 32.38, - "bfcl.overall.total_cost_usd": 6.91, - "bfcl.overall.latency_mean_s": 0.92, - "bfcl.overall.latency_std_s": 6.79, - "bfcl.overall.latency_p95_s": 2.02, - "bfcl.non_live.ast_accuracy": 89.69, - "bfcl.non_live.simple_ast_accuracy": 78.75, - "bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl.live.live_accuracy": 79.05, - "bfcl.live.live_simple_ast_accuracy": 81.4, - "bfcl.live.live_multiple_ast_accuracy": 78.54, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 14.75, - "bfcl.multi_turn.base_accuracy": 20.5, - "bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl.multi_turn.miss_parameter_accuracy": 9.5, - "bfcl.multi_turn.long_context_accuracy": 12.0, - "bfcl.web_search.accuracy": 7.5, - "bfcl.web_search.base_accuracy": 9.0, - "bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl.memory.accuracy": 15.05, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 11.61, - "bfcl.memory.recursive_summarization_accuracy": 30.97, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 65.73, - "bfcl.format_sensitivity.max_delta": 50.0, - "bfcl.format_sensitivity.stddev": 13.57 - } - }, - { - "model_id": "mistralai/open-mistral-nemo-2407-fc", - "name": "Open-Mistral-Nemo-2407 (FC)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 78.0, - "bfcl.overall.overall_accuracy": 27.63, - "bfcl.overall.total_cost_usd": 8.12, - "bfcl.overall.latency_mean_s": 1.07, - "bfcl.overall.latency_std_s": 11.93, - "bfcl.overall.latency_p95_s": 1.39, - "bfcl.non_live.ast_accuracy": 82.81, - "bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl.live.live_accuracy": 73.8, - "bfcl.live.live_simple_ast_accuracy": 78.68, - "bfcl.live.live_multiple_ast_accuracy": 72.84, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 7.75, - "bfcl.multi_turn.base_accuracy": 12.5, - "bfcl.multi_turn.miss_function_accuracy": 6.5, - "bfcl.multi_turn.miss_parameter_accuracy": 7.5, - "bfcl.multi_turn.long_context_accuracy": 4.5, - "bfcl.web_search.accuracy": 7.0, - "bfcl.web_search.base_accuracy": 9.0, - "bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl.memory.accuracy": 10.32, - "bfcl.memory.kv_accuracy": 8.39, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 12.9, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 61.77 - } - }, - { - "model_id": "mistralai/open-mistral-nemo-2407-prompt", - "name": "Open-Mistral-Nemo-2407 (Prompt)", - "developer": "mistralai", - "scores": { - "bfcl.overall.rank": 102.0, - "bfcl.overall.overall_accuracy": 19.31, - "bfcl.overall.total_cost_usd": 13.8, - "bfcl.overall.latency_mean_s": 0.84, - "bfcl.overall.latency_std_s": 7.05, - "bfcl.overall.latency_p95_s": 1.32, - "bfcl.non_live.ast_accuracy": 88.46, - "bfcl.non_live.simple_ast_accuracy": 79.33, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 90.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl.live.live_accuracy": 73.95, - "bfcl.live.live_simple_ast_accuracy": 78.29, - "bfcl.live.live_multiple_ast_accuracy": 73.03, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 0.75, - "bfcl.multi_turn.base_accuracy": 0.5, - "bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl.web_search.accuracy": 2.5, - "bfcl.web_search.base_accuracy": 3.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 8.6, - "bfcl.memory.kv_accuracy": 9.68, - "bfcl.memory.vector_accuracy": 9.68, - "bfcl.memory.recursive_summarization_accuracy": 6.45, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 6.28, - "bfcl.format_sensitivity.max_delta": 14.5, - "bfcl.format_sensitivity.stddev": 4.6 - } - }, - { - "model_id": "moonshotai/moonshotai-kimi-k2-instruct-fc", - "name": "Moonshotai-Kimi-K2-Instruct (FC)", - "developer": "moonshotai", - "scores": { - "bfcl.overall.rank": 11.0, - "bfcl.overall.overall_accuracy": 59.06, - "bfcl.overall.total_cost_usd": 6.19, - "bfcl.overall.latency_mean_s": 6.4, - "bfcl.overall.latency_std_s": 9.38, - "bfcl.overall.latency_p95_s": 13.78, - "bfcl.non_live.ast_accuracy": 81.6, - "bfcl.non_live.simple_ast_accuracy": 69.42, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 82.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl.live.live_accuracy": 78.68, - "bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 50.63, - "bfcl.multi_turn.base_accuracy": 62.0, - "bfcl.multi_turn.miss_function_accuracy": 41.0, - "bfcl.multi_turn.miss_parameter_accuracy": 44.5, - "bfcl.multi_turn.long_context_accuracy": 55.0, - "bfcl.web_search.accuracy": 66.5, - "bfcl.web_search.base_accuracy": 72.0, - "bfcl.web_search.no_snippet_accuracy": 61.0, - "bfcl.memory.accuracy": 29.03, - "bfcl.memory.kv_accuracy": 21.94, - "bfcl.memory.vector_accuracy": 20.0, - "bfcl.memory.recursive_summarization_accuracy": 45.16, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 87.34 - } - }, - { - "model_id": "nanbeige/nanbeige3-5-pro-thinking-fc", - "name": "Nanbeige3.5-Pro-Thinking (FC)", - "developer": "nanbeige", - "scores": { - "bfcl.overall.rank": 32.0, - "bfcl.overall.overall_accuracy": 47.68, - "bfcl.overall.total_cost_usd": 23.46, - "bfcl.overall.latency_mean_s": 21.12, - "bfcl.overall.latency_std_s": 28.61, - "bfcl.overall.latency_p95_s": 63.29, - "bfcl.non_live.ast_accuracy": 38.35, - "bfcl.non_live.simple_ast_accuracy": 43.92, - "bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 20.0, - "bfcl.live.live_accuracy": 69.95, - "bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl.live.live_multiple_ast_accuracy": 71.42, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 40.0, - "bfcl.multi_turn.base_accuracy": 56.0, - "bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl.multi_turn.miss_parameter_accuracy": 29.0, - "bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl.web_search.accuracy": 42.0, - "bfcl.web_search.base_accuracy": 47.0, - "bfcl.web_search.no_snippet_accuracy": 37.0, - "bfcl.memory.accuracy": 45.16, - "bfcl.memory.kv_accuracy": 38.06, - "bfcl.memory.vector_accuracy": 58.06, - "bfcl.memory.recursive_summarization_accuracy": 39.35, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 74.2 - } - }, - { - "model_id": "nanbeige/nanbeige4-3b-thinking-2511-fc", - "name": "Nanbeige4-3B-Thinking-2511 (FC)", - "developer": "nanbeige", - "scores": { - "bfcl.overall.rank": 25.0, - "bfcl.overall.overall_accuracy": 51.4, - "bfcl.overall.total_cost_usd": 14.14, - "bfcl.overall.latency_mean_s": 13.46, - "bfcl.overall.latency_std_s": 26.41, - "bfcl.overall.latency_p95_s": 37.45, - "bfcl.non_live.ast_accuracy": 81.58, - "bfcl.non_live.simple_ast_accuracy": 63.83, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 84.5, - "bfcl.live.live_accuracy": 79.42, - "bfcl.live.live_simple_ast_accuracy": 86.05, - "bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 51.12, - "bfcl.multi_turn.base_accuracy": 58.5, - "bfcl.multi_turn.miss_function_accuracy": 54.0, - "bfcl.multi_turn.miss_parameter_accuracy": 45.0, - "bfcl.multi_turn.long_context_accuracy": 47.0, - "bfcl.web_search.accuracy": 21.5, - "bfcl.web_search.base_accuracy": 31.0, - "bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl.memory.accuracy": 36.77, - "bfcl.memory.kv_accuracy": 31.61, - "bfcl.memory.vector_accuracy": 34.19, - "bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 83.09 - } - }, - { - "model_id": "nvidia/llama-3-1-nemotron-ultra-253b-v1-fc", - "name": "Llama-3.1-Nemotron-Ultra-253B-v1 (FC)", - "developer": "nvidia", - "scores": { - "bfcl.overall.rank": 108.0, - "bfcl.overall.overall_accuracy": 10.0, - "bfcl.overall.total_cost_usd": 0.72, - "bfcl.overall.latency_mean_s": 1.42, - "bfcl.overall.latency_std_s": 1.84, - "bfcl.overall.latency_p95_s": 2.4, - "bfcl.non_live.ast_accuracy": 0.0, - "bfcl.non_live.simple_ast_accuracy": 0.0, - "bfcl.non_live.multiple_ast_accuracy": 0.0, - "bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl.live.live_accuracy": 0.0, - "bfcl.live.live_simple_ast_accuracy": 0.0, - "bfcl.live.live_multiple_ast_accuracy": 0.0, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 0.0, - "bfcl.multi_turn.base_accuracy": 0.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 0.0, - "bfcl.memory.kv_accuracy": 0.0, - "bfcl.memory.vector_accuracy": 0.0, - "bfcl.memory.recursive_summarization_accuracy": 0.0, - "bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl.relevance.irrelevance_detection_accuracy": 100.0 - } - }, - { - "model_id": "openai/gpt-4-1-2025-04-14-fc", - "name": "GPT-4.1-2025-04-14 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 20.0, - "bfcl.overall.overall_accuracy": 53.96, - "bfcl.overall.total_cost_usd": 100.75, - "bfcl.overall.latency_mean_s": 1.63, - "bfcl.overall.latency_std_s": 3.05, - "bfcl.overall.latency_p95_s": 4.01, - "bfcl.non_live.ast_accuracy": 82.79, - "bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl.live.live_accuracy": 69.95, - "bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl.live.live_multiple_ast_accuracy": 70.28, - "bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 38.88, - "bfcl.multi_turn.base_accuracy": 47.5, - "bfcl.multi_turn.miss_function_accuracy": 32.5, - "bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl.web_search.accuracy": 68.0, - "bfcl.web_search.base_accuracy": 67.0, - "bfcl.web_search.no_snippet_accuracy": 69.0, - "bfcl.memory.accuracy": 23.87, - "bfcl.memory.kv_accuracy": 16.13, - "bfcl.memory.vector_accuracy": 18.06, - "bfcl.memory.recursive_summarization_accuracy": 37.42, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 86.52 - } - }, - { - "model_id": "openai/gpt-4-1-2025-04-14-prompt", - "name": "GPT-4.1-2025-04-14 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 45.0, - "bfcl.overall.overall_accuracy": 39.38, - "bfcl.overall.total_cost_usd": 145.85, - "bfcl.overall.latency_mean_s": 1.2, - "bfcl.overall.latency_std_s": 3.23, - "bfcl.overall.latency_p95_s": 2.53, - "bfcl.non_live.ast_accuracy": 88.69, - "bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl.live.live_accuracy": 78.9, - "bfcl.live.live_simple_ast_accuracy": 84.88, - "bfcl.live.live_multiple_ast_accuracy": 77.4, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 9.75, - "bfcl.multi_turn.base_accuracy": 10.5, - "bfcl.multi_turn.miss_function_accuracy": 11.0, - "bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl.multi_turn.long_context_accuracy": 9.5, - "bfcl.web_search.accuracy": 35.0, - "bfcl.web_search.base_accuracy": 40.0, - "bfcl.web_search.no_snippet_accuracy": 30.0, - "bfcl.memory.accuracy": 21.51, - "bfcl.memory.kv_accuracy": 9.68, - "bfcl.memory.vector_accuracy": 19.35, - "bfcl.memory.recursive_summarization_accuracy": 35.48, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 83.99, - "bfcl.format_sensitivity.max_delta": 23.5, - "bfcl.format_sensitivity.stddev": 6.18 - } - }, - { - "model_id": "openai/gpt-4-1-mini-2025-04-14-fc", - "name": "GPT-4.1-mini-2025-04-14 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 27.0, - "bfcl.overall.overall_accuracy": 50.45, - "bfcl.overall.total_cost_usd": 19.25, - "bfcl.overall.latency_mean_s": 1.32, - "bfcl.overall.latency_std_s": 3.65, - "bfcl.overall.latency_p95_s": 2.4, - "bfcl.non_live.ast_accuracy": 83.83, - "bfcl.non_live.simple_ast_accuracy": 73.33, - "bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl.non_live.parallel_ast_accuracy": 91.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 82.0, - "bfcl.live.live_accuracy": 68.84, - "bfcl.live.live_simple_ast_accuracy": 67.05, - "bfcl.live.live_multiple_ast_accuracy": 69.8, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 34.13, - "bfcl.multi_turn.base_accuracy": 43.5, - "bfcl.multi_turn.miss_function_accuracy": 22.5, - "bfcl.multi_turn.miss_parameter_accuracy": 30.5, - "bfcl.multi_turn.long_context_accuracy": 40.0, - "bfcl.web_search.accuracy": 57.0, - "bfcl.web_search.base_accuracy": 62.0, - "bfcl.web_search.no_snippet_accuracy": 52.0, - "bfcl.memory.accuracy": 26.88, - "bfcl.memory.kv_accuracy": 22.58, - "bfcl.memory.vector_accuracy": 16.13, - "bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 81.69 - } - }, - { - "model_id": "openai/gpt-4-1-mini-2025-04-14-prompt", - "name": "GPT-4.1-mini-2025-04-14 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 67.0, - "bfcl.overall.overall_accuracy": 29.73, - "bfcl.overall.total_cost_usd": 20.52, - "bfcl.overall.latency_mean_s": 1.36, - "bfcl.overall.latency_std_s": 4.5, - "bfcl.overall.latency_p95_s": 3.38, - "bfcl.non_live.ast_accuracy": 84.6, - "bfcl.non_live.simple_ast_accuracy": 74.92, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl.live.live_accuracy": 74.76, - "bfcl.live.live_simple_ast_accuracy": 80.62, - "bfcl.live.live_multiple_ast_accuracy": 73.31, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 2.5, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl.web_search.accuracy": 4.0, - "bfcl.web_search.base_accuracy": 7.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 24.3, - "bfcl.memory.kv_accuracy": 20.65, - "bfcl.memory.vector_accuracy": 13.55, - "bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 73.88, - "bfcl.format_sensitivity.max_delta": 45.0, - "bfcl.format_sensitivity.stddev": 13.33 - } - }, - { - "model_id": "openai/gpt-4-1-nano-2025-04-14-fc", - "name": "GPT-4.1-nano-2025-04-14 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 58.0, - "bfcl.overall.overall_accuracy": 33.05, - "bfcl.overall.total_cost_usd": 5.66, - "bfcl.overall.latency_mean_s": 1.44, - "bfcl.overall.latency_std_s": 10.84, - "bfcl.overall.latency_p95_s": 2.26, - "bfcl.non_live.ast_accuracy": 72.98, - "bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl.non_live.multiple_ast_accuracy": 79.5, - "bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 68.5, - "bfcl.live.live_accuracy": 60.77, - "bfcl.live.live_simple_ast_accuracy": 58.14, - "bfcl.live.live_multiple_ast_accuracy": 61.44, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 23.62, - "bfcl.multi_turn.base_accuracy": 39.5, - "bfcl.multi_turn.miss_function_accuracy": 7.5, - "bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl.multi_turn.long_context_accuracy": 30.0, - "bfcl.web_search.accuracy": 11.0, - "bfcl.web_search.base_accuracy": 13.0, - "bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl.memory.accuracy": 18.92, - "bfcl.memory.kv_accuracy": 10.32, - "bfcl.memory.vector_accuracy": 19.35, - "bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 66.0 - } - }, - { - "model_id": "openai/gpt-4-1-nano-2025-04-14-prompt", - "name": "GPT-4.1-nano-2025-04-14 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 90.0, - "bfcl.overall.overall_accuracy": 24.88, - "bfcl.overall.total_cost_usd": 7.42, - "bfcl.overall.latency_mean_s": 1.02, - "bfcl.overall.latency_std_s": 7.3, - "bfcl.overall.latency_p95_s": 1.88, - "bfcl.non_live.ast_accuracy": 72.44, - "bfcl.non_live.simple_ast_accuracy": 68.75, - "bfcl.non_live.multiple_ast_accuracy": 63.5, - "bfcl.non_live.parallel_ast_accuracy": 85.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 72.5, - "bfcl.live.live_accuracy": 50.33, - "bfcl.live.live_simple_ast_accuracy": 63.18, - "bfcl.live.live_multiple_ast_accuracy": 46.53, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 2.0, - "bfcl.multi_turn.base_accuracy": 2.5, - "bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl.multi_turn.long_context_accuracy": 2.0, - "bfcl.web_search.accuracy": 1.5, - "bfcl.web_search.base_accuracy": 2.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 16.77, - "bfcl.memory.kv_accuracy": 9.03, - "bfcl.memory.vector_accuracy": 14.19, - "bfcl.memory.recursive_summarization_accuracy": 27.1, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 83.44, - "bfcl.format_sensitivity.max_delta": 73.0, - "bfcl.format_sensitivity.stddev": 17.08 - } - }, - { - "model_id": "openai/gpt-5-2-2025-12-11-fc", - "name": "GPT-5.2-2025-12-11 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 16.0, - "bfcl.overall.overall_accuracy": 55.87, - "bfcl.overall.total_cost_usd": 85.65, - "bfcl.overall.latency_mean_s": 2.23, - "bfcl.overall.latency_std_s": 9.75, - "bfcl.overall.latency_p95_s": 5.26, - "bfcl.non_live.ast_accuracy": 81.85, - "bfcl.non_live.simple_ast_accuracy": 72.92, - "bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 77.5, - "bfcl.live.live_accuracy": 70.39, - "bfcl.live.live_simple_ast_accuracy": 71.71, - "bfcl.live.live_multiple_ast_accuracy": 70.37, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 28.12, - "bfcl.multi_turn.base_accuracy": 36.5, - "bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl.multi_turn.miss_parameter_accuracy": 27.5, - "bfcl.multi_turn.long_context_accuracy": 30.5, - "bfcl.web_search.accuracy": 75.5, - "bfcl.web_search.base_accuracy": 78.0, - "bfcl.web_search.no_snippet_accuracy": 73.0, - "bfcl.memory.accuracy": 45.81, - "bfcl.memory.kv_accuracy": 33.55, - "bfcl.memory.vector_accuracy": 43.23, - "bfcl.memory.recursive_summarization_accuracy": 60.65, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 79.42 - } - }, - { - "model_id": "openai/gpt-5-2-2025-12-11-prompt", - "name": "GPT-5.2-2025-12-11 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 38.0, - "bfcl.overall.overall_accuracy": 45.27, - "bfcl.overall.total_cost_usd": 164.58, - "bfcl.overall.latency_mean_s": 4.21, - "bfcl.overall.latency_std_s": 20.93, - "bfcl.overall.latency_p95_s": 10.58, - "bfcl.non_live.ast_accuracy": 78.29, - "bfcl.non_live.simple_ast_accuracy": 71.17, - "bfcl.non_live.multiple_ast_accuracy": 83.5, - "bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 74.5, - "bfcl.live.live_accuracy": 67.14, - "bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl.live.live_multiple_ast_accuracy": 64.58, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 43.75, - "bfcl.multi_turn.base_accuracy": 54.5, - "bfcl.multi_turn.miss_function_accuracy": 40.5, - "bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl.multi_turn.long_context_accuracy": 46.5, - "bfcl.web_search.accuracy": 40.5, - "bfcl.web_search.base_accuracy": 45.0, - "bfcl.web_search.no_snippet_accuracy": 36.0, - "bfcl.memory.accuracy": 3.87, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 1.94, - "bfcl.memory.recursive_summarization_accuracy": 7.1, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 87.26, - "bfcl.format_sensitivity.max_delta": 13.0, - "bfcl.format_sensitivity.stddev": 3.25 - } - }, - { - "model_id": "openai/gpt-5-mini-2025-08-07-fc", - "name": "GPT-5-mini-2025-08-07 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 17.0, - "bfcl.overall.overall_accuracy": 55.46, - "bfcl.overall.total_cost_usd": 22.18, - "bfcl.overall.latency_mean_s": 8.32, - "bfcl.overall.latency_std_s": 17.35, - "bfcl.overall.latency_p95_s": 19.8, - "bfcl.non_live.ast_accuracy": 69.85, - "bfcl.non_live.simple_ast_accuracy": 59.92, - "bfcl.non_live.multiple_ast_accuracy": 69.0, - "bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 70.5, - "bfcl.live.live_accuracy": 58.62, - "bfcl.live.live_simple_ast_accuracy": 62.02, - "bfcl.live.live_multiple_ast_accuracy": 58.02, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 45.83, - "bfcl.multi_turn.accuracy": 27.5, - "bfcl.multi_turn.base_accuracy": 36.5, - "bfcl.multi_turn.miss_function_accuracy": 17.0, - "bfcl.multi_turn.miss_parameter_accuracy": 23.5, - "bfcl.multi_turn.long_context_accuracy": 33.0, - "bfcl.web_search.accuracy": 82.0, - "bfcl.web_search.base_accuracy": 87.0, - "bfcl.web_search.no_snippet_accuracy": 77.0, - "bfcl.memory.accuracy": 44.3, - "bfcl.memory.kv_accuracy": 36.77, - "bfcl.memory.vector_accuracy": 43.87, - "bfcl.memory.recursive_summarization_accuracy": 52.26, - "bfcl.relevance.relevance_detection_accuracy": 62.5, - "bfcl.relevance.irrelevance_detection_accuracy": 91.01 - } - }, - { - "model_id": "openai/gpt-5-mini-2025-08-07-prompt", - "name": "GPT-5-mini-2025-08-07 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 77.0, - "bfcl.overall.overall_accuracy": 27.83, - "bfcl.overall.total_cost_usd": 82.74, - "bfcl.overall.latency_mean_s": 8.89, - "bfcl.overall.latency_std_s": 11.08, - "bfcl.overall.latency_p95_s": 19.72, - "bfcl.non_live.ast_accuracy": 68.04, - "bfcl.non_live.simple_ast_accuracy": 59.17, - "bfcl.non_live.multiple_ast_accuracy": 72.5, - "bfcl.non_live.parallel_ast_accuracy": 71.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl.live.live_accuracy": 62.55, - "bfcl.live.live_simple_ast_accuracy": 69.77, - "bfcl.live.live_multiple_ast_accuracy": 61.16, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 5.5, - "bfcl.multi_turn.base_accuracy": 5.5, - "bfcl.multi_turn.miss_function_accuracy": 5.0, - "bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl.web_search.accuracy": 8.5, - "bfcl.web_search.base_accuracy": 11.0, - "bfcl.web_search.no_snippet_accuracy": 6.0, - "bfcl.memory.accuracy": 29.25, - "bfcl.memory.kv_accuracy": 19.35, - "bfcl.memory.vector_accuracy": 29.68, - "bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 55.71, - "bfcl.format_sensitivity.max_delta": 16.0, - "bfcl.format_sensitivity.stddev": 3.78 - } - }, - { - "model_id": "openai/gpt-5-nano-2025-08-07-fc", - "name": "GPT-5-nano-2025-08-07 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 24.0, - "bfcl.overall.overall_accuracy": 51.45, - "bfcl.overall.total_cost_usd": 8.79, - "bfcl.overall.latency_mean_s": 10.36, - "bfcl.overall.latency_std_s": 10.37, - "bfcl.overall.latency_p95_s": 23.56, - "bfcl.non_live.ast_accuracy": 68.0, - "bfcl.non_live.simple_ast_accuracy": 57.0, - "bfcl.non_live.multiple_ast_accuracy": 64.5, - "bfcl.non_live.parallel_ast_accuracy": 79.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 71.5, - "bfcl.live.live_accuracy": 59.44, - "bfcl.live.live_simple_ast_accuracy": 58.91, - "bfcl.live.live_multiple_ast_accuracy": 59.83, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 34.5, - "bfcl.multi_turn.base_accuracy": 44.0, - "bfcl.multi_turn.miss_function_accuracy": 23.5, - "bfcl.multi_turn.miss_parameter_accuracy": 32.5, - "bfcl.multi_turn.long_context_accuracy": 38.0, - "bfcl.web_search.accuracy": 72.5, - "bfcl.web_search.base_accuracy": 74.0, - "bfcl.web_search.no_snippet_accuracy": 71.0, - "bfcl.memory.accuracy": 24.73, - "bfcl.memory.kv_accuracy": 18.06, - "bfcl.memory.vector_accuracy": 27.1, - "bfcl.memory.recursive_summarization_accuracy": 29.03, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 89.1 - } - }, - { - "model_id": "openai/gpt-5-nano-2025-08-07-prompt", - "name": "GPT-5-nano-2025-08-07 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 79.0, - "bfcl.overall.overall_accuracy": 27.55, - "bfcl.overall.total_cost_usd": 21.47, - "bfcl.overall.latency_mean_s": 10.67, - "bfcl.overall.latency_std_s": 7.68, - "bfcl.overall.latency_p95_s": 23.28, - "bfcl.non_live.ast_accuracy": 80.81, - "bfcl.non_live.simple_ast_accuracy": 69.25, - "bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl.live.live_accuracy": 70.69, - "bfcl.live.live_simple_ast_accuracy": 76.36, - "bfcl.live.live_multiple_ast_accuracy": 69.71, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 0.75, - "bfcl.multi_turn.base_accuracy": 1.0, - "bfcl.multi_turn.miss_function_accuracy": 1.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl.web_search.accuracy": 13.5, - "bfcl.web_search.base_accuracy": 10.0, - "bfcl.web_search.no_snippet_accuracy": 17.0, - "bfcl.memory.accuracy": 24.52, - "bfcl.memory.kv_accuracy": 20.65, - "bfcl.memory.vector_accuracy": 31.61, - "bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 45.75, - "bfcl.format_sensitivity.max_delta": 8.5, - "bfcl.format_sensitivity.stddev": 2.57 - } - }, - { - "model_id": "openai/o3-2025-04-16-fc", - "name": "o3-2025-04-16 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 30.0, - "bfcl.overall.overall_accuracy": 48.56, - "bfcl.overall.total_cost_usd": 133.45, - "bfcl.overall.latency_mean_s": 3.5, - "bfcl.overall.latency_std_s": 8.69, - "bfcl.overall.latency_p95_s": 8.39, - "bfcl.non_live.ast_accuracy": 40.38, - "bfcl.non_live.simple_ast_accuracy": 74.5, - "bfcl.non_live.multiple_ast_accuracy": 87.0, - "bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl.live.live_accuracy": 66.17, - "bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl.live.live_multiple_ast_accuracy": 67.62, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 14.75, - "bfcl.multi_turn.base_accuracy": 16.5, - "bfcl.multi_turn.miss_function_accuracy": 11.5, - "bfcl.multi_turn.miss_parameter_accuracy": 14.5, - "bfcl.multi_turn.long_context_accuracy": 16.5, - "bfcl.web_search.accuracy": 77.0, - "bfcl.web_search.base_accuracy": 79.0, - "bfcl.web_search.no_snippet_accuracy": 75.0, - "bfcl.memory.accuracy": 47.31, - "bfcl.memory.kv_accuracy": 24.52, - "bfcl.memory.vector_accuracy": 44.52, - "bfcl.memory.recursive_summarization_accuracy": 72.9, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 86.13 - } - }, - { - "model_id": "openai/o3-2025-04-16-prompt", - "name": "o3-2025-04-16 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 8.0, - "bfcl.overall.overall_accuracy": 63.05, - "bfcl.overall.total_cost_usd": 234.64, - "bfcl.overall.latency_mean_s": 4.83, - "bfcl.overall.latency_std_s": 7.01, - "bfcl.overall.latency_p95_s": 11.7, - "bfcl.non_live.ast_accuracy": 81.94, - "bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 78.0, - "bfcl.live.live_accuracy": 73.21, - "bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl.live.live_multiple_ast_accuracy": 70.75, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 62.25, - "bfcl.multi_turn.base_accuracy": 68.0, - "bfcl.multi_turn.miss_function_accuracy": 63.5, - "bfcl.multi_turn.miss_parameter_accuracy": 54.5, - "bfcl.multi_turn.long_context_accuracy": 63.0, - "bfcl.web_search.accuracy": 50.5, - "bfcl.web_search.base_accuracy": 51.0, - "bfcl.web_search.no_snippet_accuracy": 50.0, - "bfcl.memory.accuracy": 51.83, - "bfcl.memory.kv_accuracy": 33.55, - "bfcl.memory.vector_accuracy": 50.32, - "bfcl.memory.recursive_summarization_accuracy": 71.61, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 83.98, - "bfcl.format_sensitivity.max_delta": 8.5, - "bfcl.format_sensitivity.stddev": 2.75 - } - }, - { - "model_id": "openai/o4-mini-2025-04-16-fc", - "name": "o4-mini-2025-04-16 (FC)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 21.0, - "bfcl.overall.overall_accuracy": 53.24, - "bfcl.overall.total_cost_usd": 81.91, - "bfcl.overall.latency_mean_s": 3.71, - "bfcl.overall.latency_std_s": 7.18, - "bfcl.overall.latency_p95_s": 9.33, - "bfcl.non_live.ast_accuracy": 37.73, - "bfcl.non_live.simple_ast_accuracy": 66.92, - "bfcl.non_live.multiple_ast_accuracy": 84.0, - "bfcl.non_live.parallel_ast_accuracy": 0.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 0.0, - "bfcl.live.live_accuracy": 66.1, - "bfcl.live.live_simple_ast_accuracy": 69.38, - "bfcl.live.live_multiple_ast_accuracy": 67.81, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 0.0, - "bfcl.multi_turn.accuracy": 41.75, - "bfcl.multi_turn.base_accuracy": 51.0, - "bfcl.multi_turn.miss_function_accuracy": 30.0, - "bfcl.multi_turn.miss_parameter_accuracy": 40.5, - "bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl.web_search.accuracy": 75.5, - "bfcl.web_search.base_accuracy": 75.0, - "bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl.memory.accuracy": 34.19, - "bfcl.memory.kv_accuracy": 19.35, - "bfcl.memory.vector_accuracy": 24.52, - "bfcl.memory.recursive_summarization_accuracy": 58.71, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 83.91 - } - }, - { - "model_id": "openai/o4-mini-2025-04-16-prompt", - "name": "o4-mini-2025-04-16 (Prompt)", - "developer": "OpenAI", - "scores": { - "bfcl.overall.rank": 28.0, - "bfcl.overall.overall_accuracy": 50.26, - "bfcl.overall.total_cost_usd": 133.63, - "bfcl.overall.latency_mean_s": 4.47, - "bfcl.overall.latency_std_s": 5.19, - "bfcl.overall.latency_p95_s": 10.19, - "bfcl.non_live.ast_accuracy": 81.29, - "bfcl.non_live.simple_ast_accuracy": 72.67, - "bfcl.non_live.multiple_ast_accuracy": 88.0, - "bfcl.non_live.parallel_ast_accuracy": 84.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl.live.live_accuracy": 70.76, - "bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl.live.live_multiple_ast_accuracy": 68.76, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 16.62, - "bfcl.multi_turn.base_accuracy": 16.5, - "bfcl.multi_turn.miss_function_accuracy": 18.0, - "bfcl.multi_turn.miss_parameter_accuracy": 17.5, - "bfcl.multi_turn.long_context_accuracy": 14.5, - "bfcl.web_search.accuracy": 71.5, - "bfcl.web_search.base_accuracy": 73.0, - "bfcl.web_search.no_snippet_accuracy": 70.0, - "bfcl.memory.accuracy": 35.27, - "bfcl.memory.kv_accuracy": 22.58, - "bfcl.memory.vector_accuracy": 25.16, - "bfcl.memory.recursive_summarization_accuracy": 58.06, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 87.16, - "bfcl.format_sensitivity.max_delta": 9.5, - "bfcl.format_sensitivity.stddev": 2.6 - } - }, - { - "model_id": "openbmb/minicpm3-4b-fc-fc", - "name": "MiniCPM3-4B-FC (FC)", - "developer": "openbmb", - "scores": { - "bfcl.overall.rank": 86.0, - "bfcl.overall.overall_accuracy": 25.55, - "bfcl.overall.total_cost_usd": 54.05, - "bfcl.overall.latency_mean_s": 118.62, - "bfcl.overall.latency_std_s": 143.98, - "bfcl.overall.latency_p95_s": 388.67, - "bfcl.non_live.ast_accuracy": 81.75, - "bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 84.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.5, - "bfcl.live.live_accuracy": 65.21, - "bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl.live.live_multiple_ast_accuracy": 63.53, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 3.88, - "bfcl.multi_turn.base_accuracy": 6.5, - "bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl.multi_turn.miss_parameter_accuracy": 4.5, - "bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 12.04, - "bfcl.memory.kv_accuracy": 9.68, - "bfcl.memory.vector_accuracy": 15.48, - "bfcl.memory.recursive_summarization_accuracy": 10.97, - "bfcl.relevance.relevance_detection_accuracy": 68.75, - "bfcl.relevance.irrelevance_detection_accuracy": 72.84 - } - }, - { - "model_id": "openbmb/minicpm3-4b-prompt", - "name": "MiniCPM3-4B (Prompt)", - "developer": "openbmb", - "scores": { - "bfcl.overall.rank": 97.0, - "bfcl.overall.overall_accuracy": 22.08, - "bfcl.overall.total_cost_usd": 29.83, - "bfcl.overall.latency_mean_s": 31.18, - "bfcl.overall.latency_std_s": 35.61, - "bfcl.overall.latency_p95_s": 102.02, - "bfcl.non_live.ast_accuracy": 70.54, - "bfcl.non_live.simple_ast_accuracy": 66.17, - "bfcl.non_live.multiple_ast_accuracy": 77.0, - "bfcl.non_live.parallel_ast_accuracy": 70.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 69.0, - "bfcl.live.live_accuracy": 43.15, - "bfcl.live.live_simple_ast_accuracy": 47.67, - "bfcl.live.live_multiple_ast_accuracy": 42.17, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 3.5, - "bfcl.multi_turn.base_accuracy": 4.5, - "bfcl.multi_turn.miss_function_accuracy": 4.5, - "bfcl.multi_turn.miss_parameter_accuracy": 2.0, - "bfcl.multi_turn.long_context_accuracy": 3.0, - "bfcl.web_search.accuracy": 2.0, - "bfcl.web_search.base_accuracy": 2.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 9.46, - "bfcl.memory.kv_accuracy": 8.39, - "bfcl.memory.vector_accuracy": 10.32, - "bfcl.memory.recursive_summarization_accuracy": 9.68, - "bfcl.relevance.relevance_detection_accuracy": 56.25, - "bfcl.relevance.irrelevance_detection_accuracy": 73.71, - "bfcl.format_sensitivity.max_delta": 68.0, - "bfcl.format_sensitivity.stddev": 16.55 - } - }, - { - "model_id": "phronetic-ai/rzn-t-prompt", - "name": "RZN-T (Prompt)", - "developer": "phronetic-ai", - "scores": { - "bfcl.overall.rank": 96.0, - "bfcl.overall.overall_accuracy": 22.25, - "bfcl.overall.total_cost_usd": 12.31, - "bfcl.overall.latency_mean_s": 12.32, - "bfcl.overall.latency_std_s": 27.53, - "bfcl.overall.latency_p95_s": 39.84, - "bfcl.non_live.ast_accuracy": 67.94, - "bfcl.non_live.simple_ast_accuracy": 63.75, - "bfcl.non_live.multiple_ast_accuracy": 75.0, - "bfcl.non_live.parallel_ast_accuracy": 69.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 63.5, - "bfcl.live.live_accuracy": 49.74, - "bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl.live.live_multiple_ast_accuracy": 47.2, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl.multi_turn.accuracy": 2.88, - "bfcl.multi_turn.base_accuracy": 4.5, - "bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl.multi_turn.miss_parameter_accuracy": 2.5, - "bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 6.88, - "bfcl.memory.kv_accuracy": 9.03, - "bfcl.memory.vector_accuracy": 6.45, - "bfcl.memory.recursive_summarization_accuracy": 5.16, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 82.41, - "bfcl.format_sensitivity.max_delta": 63.5, - "bfcl.format_sensitivity.stddev": 25.53 - } - }, - { - "model_id": "qwen/qwen3-0-6b-fc", - "name": "Qwen3-0.6B (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 92.0, - "bfcl.overall.overall_accuracy": 23.93, - "bfcl.overall.total_cost_usd": 0.46, - "bfcl.overall.latency_mean_s": 0.68, - "bfcl.overall.latency_std_s": 8.45, - "bfcl.overall.latency_p95_s": 0.96, - "bfcl.non_live.ast_accuracy": 71.79, - "bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl.non_live.multiple_ast_accuracy": 86.0, - "bfcl.non_live.parallel_ast_accuracy": 67.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 69.5, - "bfcl.live.live_accuracy": 56.62, - "bfcl.live.live_simple_ast_accuracy": 61.24, - "bfcl.live.live_multiple_ast_accuracy": 56.13, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 3.62, - "bfcl.multi_turn.base_accuracy": 5.5, - "bfcl.multi_turn.miss_function_accuracy": 2.0, - "bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl.web_search.accuracy": 1.0, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 8.6, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 1.94, - "bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 80.84 - } - }, - { - "model_id": "qwen/qwen3-0-6b-prompt", - "name": "Qwen3-0.6B (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 94.0, - "bfcl.overall.overall_accuracy": 22.38, - "bfcl.overall.total_cost_usd": 3.65, - "bfcl.overall.latency_mean_s": 3.1, - "bfcl.overall.latency_std_s": 4.32, - "bfcl.overall.latency_p95_s": 10.31, - "bfcl.non_live.ast_accuracy": 70.0, - "bfcl.non_live.simple_ast_accuracy": 64.0, - "bfcl.non_live.multiple_ast_accuracy": 78.0, - "bfcl.non_live.parallel_ast_accuracy": 75.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 63.0, - "bfcl.live.live_accuracy": 49.37, - "bfcl.live.live_simple_ast_accuracy": 57.75, - "bfcl.live.live_multiple_ast_accuracy": 47.77, - "bfcl.live.live_parallel_ast_accuracy": 37.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 37.5, - "bfcl.multi_turn.accuracy": 1.38, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 1.5, - "bfcl.multi_turn.miss_parameter_accuracy": 1.5, - "bfcl.multi_turn.long_context_accuracy": 1.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 8.39, - "bfcl.memory.kv_accuracy": 1.29, - "bfcl.memory.vector_accuracy": 2.58, - "bfcl.memory.recursive_summarization_accuracy": 21.29, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 82.5, - "bfcl.format_sensitivity.max_delta": 60.5, - "bfcl.format_sensitivity.stddev": 24.35 - } - }, - { - "model_id": "qwen/qwen3-1-7b-fc", - "name": "Qwen3-1.7B (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 71.0, - "bfcl.overall.overall_accuracy": 28.41, - "bfcl.overall.total_cost_usd": 4.33, - "bfcl.overall.latency_mean_s": 5.12, - "bfcl.overall.latency_std_s": 7.37, - "bfcl.overall.latency_p95_s": 13.35, - "bfcl.non_live.ast_accuracy": 82.92, - "bfcl.non_live.simple_ast_accuracy": 70.67, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 80.0, - "bfcl.live.live_accuracy": 74.61, - "bfcl.live.live_simple_ast_accuracy": 76.74, - "bfcl.live.live_multiple_ast_accuracy": 74.26, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 11.0, - "bfcl.multi_turn.base_accuracy": 15.0, - "bfcl.multi_turn.miss_function_accuracy": 6.0, - "bfcl.multi_turn.miss_parameter_accuracy": 12.0, - "bfcl.multi_turn.long_context_accuracy": 11.0, - "bfcl.web_search.accuracy": 2.5, - "bfcl.web_search.base_accuracy": 3.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 6.02, - "bfcl.memory.kv_accuracy": 4.52, - "bfcl.memory.vector_accuracy": 7.74, - "bfcl.memory.recursive_summarization_accuracy": 5.81, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 76.54 - } - }, - { - "model_id": "qwen/qwen3-14b-fc", - "name": "Qwen3-14B (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 43.0, - "bfcl.overall.overall_accuracy": 41.03, - "bfcl.overall.total_cost_usd": 3.38, - "bfcl.overall.latency_mean_s": 4.5, - "bfcl.overall.latency_std_s": 18.84, - "bfcl.overall.latency_p95_s": 13.34, - "bfcl.non_live.ast_accuracy": 84.94, - "bfcl.non_live.simple_ast_accuracy": 74.75, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 80.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl.live.live_accuracy": 80.01, - "bfcl.live.live_simple_ast_accuracy": 85.66, - "bfcl.live.live_multiple_ast_accuracy": 79.01, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 34.75, - "bfcl.multi_turn.base_accuracy": 39.0, - "bfcl.multi_turn.miss_function_accuracy": 34.0, - "bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl.multi_turn.long_context_accuracy": 32.5, - "bfcl.web_search.accuracy": 10.0, - "bfcl.web_search.base_accuracy": 8.0, - "bfcl.web_search.no_snippet_accuracy": 12.0, - "bfcl.memory.accuracy": 19.57, - "bfcl.memory.kv_accuracy": 7.1, - "bfcl.memory.vector_accuracy": 16.77, - "bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 81.94 - } - }, - { - "model_id": "qwen/qwen3-14b-prompt", - "name": "Qwen3-14B (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 47.0, - "bfcl.overall.overall_accuracy": 37.77, - "bfcl.overall.total_cost_usd": 1.35, - "bfcl.overall.latency_mean_s": 1.2, - "bfcl.overall.latency_std_s": 8.5, - "bfcl.overall.latency_p95_s": 2.3, - "bfcl.non_live.ast_accuracy": 89.46, - "bfcl.non_live.simple_ast_accuracy": 76.83, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 92.0, - "bfcl.live.live_accuracy": 79.35, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 78.06, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 26.13, - "bfcl.multi_turn.base_accuracy": 16.5, - "bfcl.multi_turn.miss_function_accuracy": 37.5, - "bfcl.multi_turn.miss_parameter_accuracy": 31.0, - "bfcl.multi_turn.long_context_accuracy": 19.5, - "bfcl.web_search.accuracy": 10.5, - "bfcl.web_search.base_accuracy": 6.0, - "bfcl.web_search.no_snippet_accuracy": 15.0, - "bfcl.memory.accuracy": 11.18, - "bfcl.memory.kv_accuracy": 4.52, - "bfcl.memory.vector_accuracy": 6.45, - "bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 87.18, - "bfcl.format_sensitivity.max_delta": 14.0, - "bfcl.format_sensitivity.stddev": 3.97 - } - }, - { - "model_id": "qwen/qwen3-235b-a22b-instruct-2507-fc", - "name": "Qwen3-235B-A22B-Instruct-2507 (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 31.0, - "bfcl.overall.overall_accuracy": 47.99, - "bfcl.overall.total_cost_usd": 2.5, - "bfcl.overall.latency_mean_s": 2.57, - "bfcl.overall.latency_std_s": 2.44, - "bfcl.overall.latency_p95_s": 6.27, - "bfcl.non_live.ast_accuracy": 37.4, - "bfcl.non_live.simple_ast_accuracy": 40.58, - "bfcl.non_live.multiple_ast_accuracy": 36.5, - "bfcl.non_live.parallel_ast_accuracy": 53.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 19.5, - "bfcl.live.live_accuracy": 68.91, - "bfcl.live.live_simple_ast_accuracy": 58.53, - "bfcl.live.live_multiple_ast_accuracy": 71.6, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 45.38, - "bfcl.multi_turn.base_accuracy": 57.5, - "bfcl.multi_turn.miss_function_accuracy": 35.0, - "bfcl.multi_turn.miss_parameter_accuracy": 33.5, - "bfcl.multi_turn.long_context_accuracy": 55.5, - "bfcl.web_search.accuracy": 54.0, - "bfcl.web_search.base_accuracy": 57.0, - "bfcl.web_search.no_snippet_accuracy": 51.0, - "bfcl.memory.accuracy": 23.87, - "bfcl.memory.kv_accuracy": 7.1, - "bfcl.memory.vector_accuracy": 18.71, - "bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 81.73 - } - }, - { - "model_id": "qwen/qwen3-235b-a22b-instruct-2507-prompt", - "name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 23.0, - "bfcl.overall.overall_accuracy": 52.15, - "bfcl.overall.total_cost_usd": 3.12, - "bfcl.overall.latency_mean_s": 2.56, - "bfcl.overall.latency_std_s": 2.75, - "bfcl.overall.latency_p95_s": 7.61, - "bfcl.non_live.ast_accuracy": 90.33, - "bfcl.non_live.simple_ast_accuracy": 79.83, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 95.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl.live.live_accuracy": 78.68, - "bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl.live.live_multiple_ast_accuracy": 77.78, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 44.62, - "bfcl.multi_turn.base_accuracy": 54.0, - "bfcl.multi_turn.miss_function_accuracy": 42.5, - "bfcl.multi_turn.miss_parameter_accuracy": 31.5, - "bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl.web_search.accuracy": 50.5, - "bfcl.web_search.base_accuracy": 56.0, - "bfcl.web_search.no_snippet_accuracy": 45.0, - "bfcl.memory.accuracy": 19.35, - "bfcl.memory.kv_accuracy": 12.9, - "bfcl.memory.vector_accuracy": 11.61, - "bfcl.memory.recursive_summarization_accuracy": 33.55, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 78.89, - "bfcl.format_sensitivity.max_delta": 8.0, - "bfcl.format_sensitivity.stddev": 1.95 - } - }, - { - "model_id": "qwen/qwen3-30b-a3b-instruct-2507-fc", - "name": "Qwen3-30B-A3B-Instruct-2507 (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 41.0, - "bfcl.overall.overall_accuracy": 41.39, - "bfcl.overall.total_cost_usd": 5.62, - "bfcl.overall.latency_mean_s": 5.95, - "bfcl.overall.latency_std_s": 25.48, - "bfcl.overall.latency_p95_s": 12.7, - "bfcl.non_live.ast_accuracy": 85.77, - "bfcl.non_live.simple_ast_accuracy": 68.58, - "bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 77.94, - "bfcl.live.live_simple_ast_accuracy": 83.33, - "bfcl.live.live_multiple_ast_accuracy": 76.83, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 30.0, - "bfcl.multi_turn.base_accuracy": 43.5, - "bfcl.multi_turn.miss_function_accuracy": 10.5, - "bfcl.multi_turn.miss_parameter_accuracy": 25.0, - "bfcl.multi_turn.long_context_accuracy": 41.0, - "bfcl.web_search.accuracy": 22.5, - "bfcl.web_search.base_accuracy": 21.0, - "bfcl.web_search.no_snippet_accuracy": 24.0, - "bfcl.memory.accuracy": 17.63, - "bfcl.memory.kv_accuracy": 9.03, - "bfcl.memory.vector_accuracy": 9.03, - "bfcl.memory.recursive_summarization_accuracy": 34.84, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 79.9 - } - }, - { - "model_id": "qwen/qwen3-30b-a3b-instruct-2507-prompt", - "name": "Qwen3-30B-A3B-Instruct-2507 (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 53.0, - "bfcl.overall.overall_accuracy": 36.7, - "bfcl.overall.total_cost_usd": 1.56, - "bfcl.overall.latency_mean_s": 1.24, - "bfcl.overall.latency_std_s": 7.9, - "bfcl.overall.latency_p95_s": 2.84, - "bfcl.non_live.ast_accuracy": 88.92, - "bfcl.non_live.simple_ast_accuracy": 80.67, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 94.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.0, - "bfcl.live.live_accuracy": 78.39, - "bfcl.live.live_simple_ast_accuracy": 82.56, - "bfcl.live.live_multiple_ast_accuracy": 77.49, - "bfcl.live.live_parallel_ast_accuracy": 87.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 23.5, - "bfcl.multi_turn.base_accuracy": 33.0, - "bfcl.multi_turn.miss_function_accuracy": 16.0, - "bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl.multi_turn.long_context_accuracy": 29.0, - "bfcl.web_search.accuracy": 17.5, - "bfcl.web_search.base_accuracy": 15.0, - "bfcl.web_search.no_snippet_accuracy": 20.0, - "bfcl.memory.accuracy": 9.68, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 6.45, - "bfcl.memory.recursive_summarization_accuracy": 16.77, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 74.85, - "bfcl.format_sensitivity.max_delta": 16.0, - "bfcl.format_sensitivity.stddev": 4.13 - } - }, - { - "model_id": "qwen/qwen3-32b-fc", - "name": "Qwen3-32B (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 29.0, - "bfcl.overall.overall_accuracy": 48.71, - "bfcl.overall.total_cost_usd": 153.08, - "bfcl.overall.latency_mean_s": 169.87, - "bfcl.overall.latency_std_s": 164.27, - "bfcl.overall.latency_p95_s": 473.49, - "bfcl.non_live.ast_accuracy": 88.77, - "bfcl.non_live.simple_ast_accuracy": 75.58, - "bfcl.non_live.multiple_ast_accuracy": 94.5, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl.live.live_accuracy": 82.01, - "bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl.live.live_multiple_ast_accuracy": 80.91, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl.multi_turn.accuracy": 47.87, - "bfcl.multi_turn.base_accuracy": 56.0, - "bfcl.multi_turn.miss_function_accuracy": 52.5, - "bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl.multi_turn.long_context_accuracy": 43.0, - "bfcl.web_search.accuracy": 21.5, - "bfcl.web_search.base_accuracy": 25.0, - "bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl.memory.accuracy": 26.67, - "bfcl.memory.kv_accuracy": 12.26, - "bfcl.memory.vector_accuracy": 25.81, - "bfcl.memory.recursive_summarization_accuracy": 41.94, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 76.37 - } - }, - { - "model_id": "qwen/qwen3-32b-prompt", - "name": "Qwen3-32B (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 33.0, - "bfcl.overall.overall_accuracy": 46.78, - "bfcl.overall.total_cost_usd": 199.47, - "bfcl.overall.latency_mean_s": 167.54, - "bfcl.overall.latency_std_s": 160.5, - "bfcl.overall.latency_p95_s": 457.87, - "bfcl.non_live.ast_accuracy": 90.27, - "bfcl.non_live.simple_ast_accuracy": 79.08, - "bfcl.non_live.multiple_ast_accuracy": 97.0, - "bfcl.non_live.parallel_ast_accuracy": 93.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.5, - "bfcl.live.live_accuracy": 82.01, - "bfcl.live.live_simple_ast_accuracy": 87.21, - "bfcl.live.live_multiple_ast_accuracy": 81.2, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 43.25, - "bfcl.multi_turn.base_accuracy": 54.0, - "bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl.multi_turn.miss_parameter_accuracy": 36.5, - "bfcl.multi_turn.long_context_accuracy": 36.5, - "bfcl.web_search.accuracy": 26.0, - "bfcl.web_search.base_accuracy": 34.0, - "bfcl.web_search.no_snippet_accuracy": 18.0, - "bfcl.memory.accuracy": 15.7, - "bfcl.memory.kv_accuracy": 13.55, - "bfcl.memory.vector_accuracy": 14.19, - "bfcl.memory.recursive_summarization_accuracy": 19.35, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 82.39, - "bfcl.format_sensitivity.max_delta": 15.5, - "bfcl.format_sensitivity.stddev": 3.75 - } - }, - { - "model_id": "qwen/qwen3-4b-instruct-2507-fc", - "name": "Qwen3-4B-Instruct-2507 (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 54.0, - "bfcl.overall.overall_accuracy": 35.68, - "bfcl.overall.total_cost_usd": 6.37, - "bfcl.overall.latency_mean_s": 7.61, - "bfcl.overall.latency_std_s": 20.36, - "bfcl.overall.latency_p95_s": 49.18, - "bfcl.non_live.ast_accuracy": 87.88, - "bfcl.non_live.simple_ast_accuracy": 75.5, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl.live.live_accuracy": 76.39, - "bfcl.live.live_simple_ast_accuracy": 79.07, - "bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 66.67, - "bfcl.multi_turn.accuracy": 22.12, - "bfcl.multi_turn.base_accuracy": 26.5, - "bfcl.multi_turn.miss_function_accuracy": 21.0, - "bfcl.multi_turn.miss_parameter_accuracy": 15.5, - "bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl.web_search.accuracy": 3.0, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 17.63, - "bfcl.memory.kv_accuracy": 16.13, - "bfcl.memory.vector_accuracy": 12.26, - "bfcl.memory.recursive_summarization_accuracy": 24.52, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 84.93 - } - }, - { - "model_id": "qwen/qwen3-4b-instruct-2507-prompt", - "name": "Qwen3-4B-Instruct-2507 (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 55.0, - "bfcl.overall.overall_accuracy": 35.52, - "bfcl.overall.total_cost_usd": 53.66, - "bfcl.overall.latency_mean_s": 44.7, - "bfcl.overall.latency_std_s": 163.79, - "bfcl.overall.latency_p95_s": 208.06, - "bfcl.non_live.ast_accuracy": 86.44, - "bfcl.non_live.simple_ast_accuracy": 77.25, - "bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 74.69, - "bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl.live.live_multiple_ast_accuracy": 74.17, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 20.5, - "bfcl.multi_turn.base_accuracy": 24.5, - "bfcl.multi_turn.miss_function_accuracy": 21.5, - "bfcl.multi_turn.miss_parameter_accuracy": 16.0, - "bfcl.multi_turn.long_context_accuracy": 20.0, - "bfcl.web_search.accuracy": 4.5, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 5.0, - "bfcl.memory.accuracy": 23.87, - "bfcl.memory.kv_accuracy": 12.9, - "bfcl.memory.vector_accuracy": 14.19, - "bfcl.memory.recursive_summarization_accuracy": 44.52, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 75.87, - "bfcl.format_sensitivity.max_delta": 18.0, - "bfcl.format_sensitivity.stddev": 5.22 - } - }, - { - "model_id": "qwen/qwen3-8b-fc", - "name": "Qwen3-8B (FC)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 39.0, - "bfcl.overall.overall_accuracy": 42.57, - "bfcl.overall.total_cost_usd": 43.32, - "bfcl.overall.latency_mean_s": 51.36, - "bfcl.overall.latency_std_s": 76.14, - "bfcl.overall.latency_p95_s": 188.98, - "bfcl.non_live.ast_accuracy": 87.58, - "bfcl.non_live.simple_ast_accuracy": 72.83, - "bfcl.non_live.multiple_ast_accuracy": 96.5, - "bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl.live.live_accuracy": 80.53, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 79.68, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 41.75, - "bfcl.multi_turn.base_accuracy": 50.5, - "bfcl.multi_turn.miss_function_accuracy": 42.0, - "bfcl.multi_turn.miss_parameter_accuracy": 40.0, - "bfcl.multi_turn.long_context_accuracy": 34.5, - "bfcl.web_search.accuracy": 12.0, - "bfcl.web_search.base_accuracy": 15.0, - "bfcl.web_search.no_snippet_accuracy": 9.0, - "bfcl.memory.accuracy": 14.62, - "bfcl.memory.kv_accuracy": 5.16, - "bfcl.memory.vector_accuracy": 7.1, - "bfcl.memory.recursive_summarization_accuracy": 31.61, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 79.07 - } - }, - { - "model_id": "qwen/qwen3-8b-prompt", - "name": "Qwen3-8B (Prompt)", - "developer": "qwen", - "scores": { - "bfcl.overall.rank": 44.0, - "bfcl.overall.overall_accuracy": 40.43, - "bfcl.overall.total_cost_usd": 63.95, - "bfcl.overall.latency_mean_s": 54.17, - "bfcl.overall.latency_std_s": 79.9, - "bfcl.overall.latency_p95_s": 194.15, - "bfcl.non_live.ast_accuracy": 88.56, - "bfcl.non_live.simple_ast_accuracy": 75.25, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 94.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 80.09, - "bfcl.live.live_simple_ast_accuracy": 84.5, - "bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl.live.live_parallel_ast_accuracy": 93.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 33.38, - "bfcl.multi_turn.base_accuracy": 41.5, - "bfcl.multi_turn.miss_function_accuracy": 38.5, - "bfcl.multi_turn.miss_parameter_accuracy": 27.0, - "bfcl.multi_turn.long_context_accuracy": 26.5, - "bfcl.web_search.accuracy": 13.5, - "bfcl.web_search.base_accuracy": 19.0, - "bfcl.web_search.no_snippet_accuracy": 8.0, - "bfcl.memory.accuracy": 13.12, - "bfcl.memory.kv_accuracy": 3.87, - "bfcl.memory.vector_accuracy": 10.32, - "bfcl.memory.recursive_summarization_accuracy": 25.16, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 82.27, - "bfcl.format_sensitivity.max_delta": 16.5, - "bfcl.format_sensitivity.stddev": 5.09 - } - }, - { - "model_id": "salesforce/xlam-2-1b-fc-r-fc", - "name": "xLAM-2-1b-fc-r (FC)", - "developer": "salesforce", - "scores": { - "bfcl.overall.rank": 65.0, - "bfcl.overall.overall_accuracy": 30.44, - "bfcl.overall.total_cost_usd": 2.79, - "bfcl.overall.latency_mean_s": 2.84, - "bfcl.overall.latency_std_s": 2.35, - "bfcl.overall.latency_p95_s": 6.52, - "bfcl.non_live.ast_accuracy": 69.04, - "bfcl.non_live.simple_ast_accuracy": 64.17, - "bfcl.non_live.multiple_ast_accuracy": 82.5, - "bfcl.non_live.parallel_ast_accuracy": 73.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 56.0, - "bfcl.live.live_accuracy": 55.14, - "bfcl.live.live_simple_ast_accuracy": 68.22, - "bfcl.live.live_multiple_ast_accuracy": 52.8, - "bfcl.live.live_parallel_ast_accuracy": 43.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 25.0, - "bfcl.multi_turn.accuracy": 36.0, - "bfcl.multi_turn.base_accuracy": 45.5, - "bfcl.multi_turn.miss_function_accuracy": 36.0, - "bfcl.multi_turn.miss_parameter_accuracy": 37.0, - "bfcl.multi_turn.long_context_accuracy": 25.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 3.87, - "bfcl.memory.kv_accuracy": 3.87, - "bfcl.memory.vector_accuracy": 3.87, - "bfcl.memory.recursive_summarization_accuracy": 3.87, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 64.47 - } - }, - { - "model_id": "salesforce/xlam-2-32b-fc-r-fc", - "name": "xLAM-2-32b-fc-r (FC)", - "developer": "salesforce", - "scores": { - "bfcl.overall.rank": 18.0, - "bfcl.overall.overall_accuracy": 54.66, - "bfcl.overall.total_cost_usd": 6.0, - "bfcl.overall.latency_mean_s": 6.94, - "bfcl.overall.latency_std_s": 8.21, - "bfcl.overall.latency_p95_s": 17.66, - "bfcl.non_live.ast_accuracy": 89.6, - "bfcl.non_live.simple_ast_accuracy": 80.42, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 91.0, - "bfcl.live.live_accuracy": 75.5, - "bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl.live.live_multiple_ast_accuracy": 74.64, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 69.5, - "bfcl.multi_turn.base_accuracy": 81.5, - "bfcl.multi_turn.miss_function_accuracy": 72.5, - "bfcl.multi_turn.miss_parameter_accuracy": 67.5, - "bfcl.multi_turn.long_context_accuracy": 56.5, - "bfcl.web_search.accuracy": 25.5, - "bfcl.web_search.base_accuracy": 37.0, - "bfcl.web_search.no_snippet_accuracy": 14.0, - "bfcl.memory.accuracy": 20.86, - "bfcl.memory.kv_accuracy": 6.45, - "bfcl.memory.vector_accuracy": 10.32, - "bfcl.memory.recursive_summarization_accuracy": 45.81, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 80.23 - } - }, - { - "model_id": "salesforce/xlam-2-3b-fc-r-fc", - "name": "xLAM-2-3b-fc-r (FC)", - "developer": "salesforce", - "scores": { - "bfcl.overall.rank": 42.0, - "bfcl.overall.overall_accuracy": 41.22, - "bfcl.overall.total_cost_usd": 3.36, - "bfcl.overall.latency_mean_s": 3.8, - "bfcl.overall.latency_std_s": 3.59, - "bfcl.overall.latency_p95_s": 8.79, - "bfcl.non_live.ast_accuracy": 82.96, - "bfcl.non_live.simple_ast_accuracy": 75.33, - "bfcl.non_live.multiple_ast_accuracy": 91.0, - "bfcl.non_live.parallel_ast_accuracy": 86.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 79.0, - "bfcl.live.live_accuracy": 62.92, - "bfcl.live.live_simple_ast_accuracy": 73.26, - "bfcl.live.live_multiple_ast_accuracy": 60.68, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 50.0, - "bfcl.multi_turn.accuracy": 58.38, - "bfcl.multi_turn.base_accuracy": 71.5, - "bfcl.multi_turn.miss_function_accuracy": 59.0, - "bfcl.multi_turn.miss_parameter_accuracy": 57.5, - "bfcl.multi_turn.long_context_accuracy": 45.5, - "bfcl.web_search.accuracy": 2.5, - "bfcl.web_search.base_accuracy": 3.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 11.4, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 5.81, - "bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 63.45 - } - }, - { - "model_id": "salesforce/xlam-2-70b-fc-r-fc", - "name": "xLAM-2-70b-fc-r (FC)", - "developer": "salesforce", - "scores": { - "bfcl.overall.rank": 22.0, - "bfcl.overall.overall_accuracy": 53.07, - "bfcl.overall.total_cost_usd": 25.1, - "bfcl.overall.latency_mean_s": 28.06, - "bfcl.overall.latency_std_s": 68.77, - "bfcl.overall.latency_p95_s": 91.21, - "bfcl.non_live.ast_accuracy": 88.44, - "bfcl.non_live.simple_ast_accuracy": 78.25, - "bfcl.non_live.multiple_ast_accuracy": 94.0, - "bfcl.non_live.parallel_ast_accuracy": 92.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 72.17, - "bfcl.live.live_simple_ast_accuracy": 77.91, - "bfcl.live.live_multiple_ast_accuracy": 71.13, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 77.38, - "bfcl.multi_turn.base_accuracy": 82.5, - "bfcl.multi_turn.miss_function_accuracy": 77.0, - "bfcl.multi_turn.miss_parameter_accuracy": 74.0, - "bfcl.multi_turn.long_context_accuracy": 76.0, - "bfcl.web_search.accuracy": 15.0, - "bfcl.web_search.base_accuracy": 17.0, - "bfcl.web_search.no_snippet_accuracy": 13.0, - "bfcl.memory.accuracy": 14.41, - "bfcl.memory.kv_accuracy": 2.58, - "bfcl.memory.vector_accuracy": 10.97, - "bfcl.memory.recursive_summarization_accuracy": 29.68, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 79.11 - } - }, - { - "model_id": "salesforce/xlam-2-8b-fc-r-fc", - "name": "xLAM-2-8b-fc-r (FC)", - "developer": "salesforce", - "scores": { - "bfcl.overall.rank": 34.0, - "bfcl.overall.overall_accuracy": 46.68, - "bfcl.overall.total_cost_usd": 20.92, - "bfcl.overall.latency_mean_s": 22.65, - "bfcl.overall.latency_std_s": 46.92, - "bfcl.overall.latency_p95_s": 108.81, - "bfcl.non_live.ast_accuracy": 84.58, - "bfcl.non_live.simple_ast_accuracy": 73.83, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 87.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 83.5, - "bfcl.live.live_accuracy": 67.95, - "bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 70.0, - "bfcl.multi_turn.base_accuracy": 76.0, - "bfcl.multi_turn.miss_function_accuracy": 72.0, - "bfcl.multi_turn.miss_parameter_accuracy": 65.0, - "bfcl.multi_turn.long_context_accuracy": 67.0, - "bfcl.web_search.accuracy": 6.5, - "bfcl.web_search.base_accuracy": 11.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 13.98, - "bfcl.memory.kv_accuracy": 5.81, - "bfcl.memory.vector_accuracy": 15.48, - "bfcl.memory.recursive_summarization_accuracy": 20.65, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 63.28 - } - }, - { - "model_id": "speakleash-ack-cyfronet-agh/bielik-11b-v2-3-instruct-prompt", - "name": "Bielik-11B-v2.3-Instruct (Prompt)", - "developer": "speakleash-ack-cyfronet-agh", - "scores": { - "bfcl.overall.rank": 99.0, - "bfcl.overall.overall_accuracy": 21.9, - "bfcl.overall.total_cost_usd": 22.44, - "bfcl.overall.latency_mean_s": 23.75, - "bfcl.overall.latency_std_s": 61.76, - "bfcl.overall.latency_p95_s": 72.8, - "bfcl.non_live.ast_accuracy": 81.5, - "bfcl.non_live.simple_ast_accuracy": 73.0, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 85.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 75.5, - "bfcl.live.live_accuracy": 67.8, - "bfcl.live.live_simple_ast_accuracy": 75.58, - "bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 58.33, - "bfcl.multi_turn.accuracy": 2.62, - "bfcl.multi_turn.base_accuracy": 4.5, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 3.0, - "bfcl.multi_turn.long_context_accuracy": 2.5, - "bfcl.web_search.accuracy": 1.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 2.0, - "bfcl.memory.accuracy": 11.4, - "bfcl.memory.kv_accuracy": 7.1, - "bfcl.memory.vector_accuracy": 4.52, - "bfcl.memory.recursive_summarization_accuracy": 22.58, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 36.01, - "bfcl.format_sensitivity.max_delta": 35.0, - "bfcl.format_sensitivity.stddev": 9.74 - } - }, - { - "model_id": "tii-uae/falcon3-10b-instruct-fc", - "name": "Falcon3-10B-Instruct (FC)", - "developer": "tii-uae", - "scores": { - "bfcl.overall.rank": 82.0, - "bfcl.overall.overall_accuracy": 27.01, - "bfcl.overall.total_cost_usd": 52.59, - "bfcl.overall.latency_mean_s": 69.27, - "bfcl.overall.latency_std_s": 92.22, - "bfcl.overall.latency_p95_s": 190.96, - "bfcl.non_live.ast_accuracy": 85.0, - "bfcl.non_live.simple_ast_accuracy": 70.5, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 87.5, - "bfcl.live.live_accuracy": 75.43, - "bfcl.live.live_simple_ast_accuracy": 77.13, - "bfcl.live.live_multiple_ast_accuracy": 76.16, - "bfcl.live.live_parallel_ast_accuracy": 50.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 41.67, - "bfcl.multi_turn.accuracy": 6.5, - "bfcl.multi_turn.base_accuracy": 6.5, - "bfcl.multi_turn.miss_function_accuracy": 9.5, - "bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl.multi_turn.long_context_accuracy": 5.0, - "bfcl.web_search.accuracy": 1.5, - "bfcl.web_search.base_accuracy": 2.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 27.53, - "bfcl.memory.kv_accuracy": 12.26, - "bfcl.memory.vector_accuracy": 19.35, - "bfcl.memory.recursive_summarization_accuracy": 50.97, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 32.09 - } - }, - { - "model_id": "tii-uae/falcon3-1b-instruct-fc", - "name": "Falcon3-1B-Instruct (FC)", - "developer": "tii-uae", - "scores": { - "bfcl.overall.rank": 106.0, - "bfcl.overall.overall_accuracy": 11.08, - "bfcl.overall.total_cost_usd": 1.72, - "bfcl.overall.latency_mean_s": 5.23, - "bfcl.overall.latency_std_s": 14.34, - "bfcl.overall.latency_p95_s": 11.48, - "bfcl.non_live.ast_accuracy": 9.02, - "bfcl.non_live.simple_ast_accuracy": 2.58, - "bfcl.non_live.multiple_ast_accuracy": 6.0, - "bfcl.non_live.parallel_ast_accuracy": 18.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 9.5, - "bfcl.live.live_accuracy": 2.89, - "bfcl.live.live_simple_ast_accuracy": 4.26, - "bfcl.live.live_multiple_ast_accuracy": 2.37, - "bfcl.live.live_parallel_ast_accuracy": 0.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 12.5, - "bfcl.multi_turn.accuracy": 0.0, - "bfcl.multi_turn.base_accuracy": 0.0, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.0, - "bfcl.multi_turn.long_context_accuracy": 0.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 5.81, - "bfcl.memory.kv_accuracy": 5.16, - "bfcl.memory.vector_accuracy": 7.74, - "bfcl.memory.recursive_summarization_accuracy": 4.52, - "bfcl.relevance.relevance_detection_accuracy": 0.0, - "bfcl.relevance.irrelevance_detection_accuracy": 87.3 - } - }, - { - "model_id": "tii-uae/falcon3-3b-instruct-fc", - "name": "Falcon3-3B-Instruct (FC)", - "developer": "tii-uae", - "scores": { - "bfcl.overall.rank": 104.0, - "bfcl.overall.overall_accuracy": 16.25, - "bfcl.overall.total_cost_usd": 36.7, - "bfcl.overall.latency_mean_s": 38.52, - "bfcl.overall.latency_std_s": 107.47, - "bfcl.overall.latency_p95_s": 103.62, - "bfcl.non_live.ast_accuracy": 54.62, - "bfcl.non_live.simple_ast_accuracy": 56.5, - "bfcl.non_live.multiple_ast_accuracy": 69.5, - "bfcl.non_live.parallel_ast_accuracy": 67.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 25.5, - "bfcl.live.live_accuracy": 54.48, - "bfcl.live.live_simple_ast_accuracy": 57.36, - "bfcl.live.live_multiple_ast_accuracy": 54.7, - "bfcl.live.live_parallel_ast_accuracy": 25.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 33.33, - "bfcl.multi_turn.accuracy": 1.0, - "bfcl.multi_turn.base_accuracy": 1.5, - "bfcl.multi_turn.miss_function_accuracy": 0.5, - "bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl.multi_turn.long_context_accuracy": 1.5, - "bfcl.web_search.accuracy": 1.0, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 7.74, - "bfcl.memory.kv_accuracy": 6.45, - "bfcl.memory.vector_accuracy": 8.39, - "bfcl.memory.recursive_summarization_accuracy": 8.39, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 32.92 - } - }, - { - "model_id": "tii-uae/falcon3-7b-instruct-fc", - "name": "Falcon3-7B-Instruct (FC)", - "developer": "tii-uae", - "scores": { - "bfcl.overall.rank": 91.0, - "bfcl.overall.overall_accuracy": 24.03, - "bfcl.overall.total_cost_usd": 73.61, - "bfcl.overall.latency_mean_s": 93.11, - "bfcl.overall.latency_std_s": 117.8, - "bfcl.overall.latency_p95_s": 315.7, - "bfcl.non_live.ast_accuracy": 82.69, - "bfcl.non_live.simple_ast_accuracy": 65.75, - "bfcl.non_live.multiple_ast_accuracy": 89.0, - "bfcl.non_live.parallel_ast_accuracy": 87.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.0, - "bfcl.live.live_accuracy": 68.32, - "bfcl.live.live_simple_ast_accuracy": 74.81, - "bfcl.live.live_multiple_ast_accuracy": 66.76, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 5.0, - "bfcl.multi_turn.base_accuracy": 7.0, - "bfcl.multi_turn.miss_function_accuracy": 4.0, - "bfcl.multi_turn.miss_parameter_accuracy": 5.0, - "bfcl.multi_turn.long_context_accuracy": 4.0, - "bfcl.web_search.accuracy": 0.5, - "bfcl.web_search.base_accuracy": 1.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 20.65, - "bfcl.memory.kv_accuracy": 10.32, - "bfcl.memory.vector_accuracy": 12.9, - "bfcl.memory.recursive_summarization_accuracy": 38.71, - "bfcl.relevance.relevance_detection_accuracy": 100.0, - "bfcl.relevance.irrelevance_detection_accuracy": 31.99 - } - }, - { - "model_id": "uiuc-oumi/coalm-70b", - "name": "CoALM-70B", - "developer": "uiuc-oumi", - "scores": { - "bfcl.overall.rank": 74.0, - "bfcl.overall.overall_accuracy": 27.99, - "bfcl.overall.total_cost_usd": 19.89, - "bfcl.overall.latency_mean_s": 16.22, - "bfcl.overall.latency_std_s": 59.91, - "bfcl.overall.latency_p95_s": 36.0, - "bfcl.non_live.ast_accuracy": 83.44, - "bfcl.non_live.simple_ast_accuracy": 70.25, - "bfcl.non_live.multiple_ast_accuracy": 92.0, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 83.0, - "bfcl.live.live_accuracy": 67.28, - "bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl.live.live_multiple_ast_accuracy": 66.57, - "bfcl.live.live_parallel_ast_accuracy": 68.75, - "bfcl.live.live_parallel_multiple_ast_accuracy": 62.5, - "bfcl.multi_turn.accuracy": 10.62, - "bfcl.multi_turn.base_accuracy": 11.0, - "bfcl.multi_turn.miss_function_accuracy": 14.0, - "bfcl.multi_turn.miss_parameter_accuracy": 9.0, - "bfcl.multi_turn.long_context_accuracy": 8.5, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 5.81, - "bfcl.memory.kv_accuracy": 9.03, - "bfcl.memory.vector_accuracy": 5.16, - "bfcl.memory.recursive_summarization_accuracy": 3.23, - "bfcl.relevance.relevance_detection_accuracy": 93.75, - "bfcl.relevance.irrelevance_detection_accuracy": 85.65, - "bfcl.format_sensitivity.max_delta": 72.0, - "bfcl.format_sensitivity.stddev": 27.76 - } - }, - { - "model_id": "uiuc-oumi/coalm-8b", - "name": "CoALM-8B", - "developer": "uiuc-oumi", - "scores": { - "bfcl.overall.rank": 84.0, - "bfcl.overall.overall_accuracy": 26.81, - "bfcl.overall.total_cost_usd": 25.33, - "bfcl.overall.latency_mean_s": 20.36, - "bfcl.overall.latency_std_s": 73.74, - "bfcl.overall.latency_p95_s": 138.04, - "bfcl.non_live.ast_accuracy": 84.87, - "bfcl.non_live.simple_ast_accuracy": 69.5, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 88.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 88.5, - "bfcl.live.live_accuracy": 66.77, - "bfcl.live.live_simple_ast_accuracy": 70.54, - "bfcl.live.live_multiple_ast_accuracy": 66.19, - "bfcl.live.live_parallel_ast_accuracy": 62.5, - "bfcl.live.live_parallel_multiple_ast_accuracy": 54.17, - "bfcl.multi_turn.accuracy": 8.0, - "bfcl.multi_turn.base_accuracy": 10.0, - "bfcl.multi_turn.miss_function_accuracy": 7.0, - "bfcl.multi_turn.miss_parameter_accuracy": 8.0, - "bfcl.multi_turn.long_context_accuracy": 7.0, - "bfcl.web_search.accuracy": 0.0, - "bfcl.web_search.base_accuracy": 0.0, - "bfcl.web_search.no_snippet_accuracy": 0.0, - "bfcl.memory.accuracy": 2.8, - "bfcl.memory.kv_accuracy": 3.23, - "bfcl.memory.vector_accuracy": 3.87, - "bfcl.memory.recursive_summarization_accuracy": 1.29, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 86.9, - "bfcl.format_sensitivity.max_delta": 79.0, - "bfcl.format_sensitivity.stddev": 34.18 - } - }, - { - "model_id": "writer/palmyra-x-004-fc", - "name": "palmyra-x-004 (FC)", - "developer": "writer", - "scores": { - "bfcl.overall.rank": 76.0, - "bfcl.overall.overall_accuracy": 27.87, - "bfcl.overall.total_cost_usd": 178.15, - "bfcl.overall.latency_mean_s": 3.71, - "bfcl.overall.latency_std_s": 7.62, - "bfcl.overall.latency_p95_s": 8.04, - "bfcl.non_live.ast_accuracy": 87.46, - "bfcl.non_live.simple_ast_accuracy": 71.33, - "bfcl.non_live.multiple_ast_accuracy": 96.0, - "bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 77.87, - "bfcl.live.live_simple_ast_accuracy": 79.46, - "bfcl.live.live_multiple_ast_accuracy": 77.97, - "bfcl.live.live_parallel_ast_accuracy": 56.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 0.38, - "bfcl.multi_turn.base_accuracy": 0.5, - "bfcl.multi_turn.miss_function_accuracy": 0.0, - "bfcl.multi_turn.miss_parameter_accuracy": 0.5, - "bfcl.multi_turn.long_context_accuracy": 0.5, - "bfcl.web_search.accuracy": 2.5, - "bfcl.web_search.base_accuracy": 4.0, - "bfcl.web_search.no_snippet_accuracy": 1.0, - "bfcl.memory.accuracy": 13.12, - "bfcl.memory.kv_accuracy": 6.45, - "bfcl.memory.vector_accuracy": 14.19, - "bfcl.memory.recursive_summarization_accuracy": 18.71, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 80.99 - } - }, - { - "model_id": "xai/grok-4-0709-fc", - "name": "Grok-4-0709 (FC)", - "developer": "xAI", - "scores": { - "bfcl.overall.rank": 10.0, - "bfcl.overall.overall_accuracy": 61.38, - "bfcl.overall.total_cost_usd": 355.17, - "bfcl.overall.latency_mean_s": 15.49, - "bfcl.overall.latency_std_s": 26.22, - "bfcl.overall.latency_p95_s": 44.28, - "bfcl.non_live.ast_accuracy": 85.38, - "bfcl.non_live.simple_ast_accuracy": 73.5, - "bfcl.non_live.multiple_ast_accuracy": 92.5, - "bfcl.non_live.parallel_ast_accuracy": 88.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 87.0, - "bfcl.live.live_accuracy": 75.57, - "bfcl.live.live_simple_ast_accuracy": 82.17, - "bfcl.live.live_multiple_ast_accuracy": 73.88, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 79.17, - "bfcl.multi_turn.accuracy": 33.88, - "bfcl.multi_turn.base_accuracy": 44.0, - "bfcl.multi_turn.miss_function_accuracy": 19.0, - "bfcl.multi_turn.miss_parameter_accuracy": 28.5, - "bfcl.multi_turn.long_context_accuracy": 44.0, - "bfcl.web_search.accuracy": 82.0, - "bfcl.web_search.base_accuracy": 80.0, - "bfcl.web_search.no_snippet_accuracy": 84.0, - "bfcl.memory.accuracy": 55.91, - "bfcl.memory.kv_accuracy": 57.42, - "bfcl.memory.vector_accuracy": 58.71, - "bfcl.memory.recursive_summarization_accuracy": 51.61, - "bfcl.relevance.relevance_detection_accuracy": 87.5, - "bfcl.relevance.irrelevance_detection_accuracy": 75.4 - } - }, - { - "model_id": "xai/grok-4-0709-prompt", - "name": "Grok-4-0709 (Prompt)", - "developer": "xAI", - "scores": { - "bfcl.overall.rank": 9.0, - "bfcl.overall.overall_accuracy": 62.97, - "bfcl.overall.total_cost_usd": 348.19, - "bfcl.overall.latency_mean_s": 30.38, - "bfcl.overall.latency_std_s": 36.19, - "bfcl.overall.latency_p95_s": 101.54, - "bfcl.non_live.ast_accuracy": 82.75, - "bfcl.non_live.simple_ast_accuracy": 67.0, - "bfcl.non_live.multiple_ast_accuracy": 93.5, - "bfcl.non_live.parallel_ast_accuracy": 89.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 81.5, - "bfcl.live.live_accuracy": 72.54, - "bfcl.live.live_simple_ast_accuracy": 81.78, - "bfcl.live.live_multiple_ast_accuracy": 70.18, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 47.0, - "bfcl.multi_turn.base_accuracy": 55.5, - "bfcl.multi_turn.miss_function_accuracy": 46.0, - "bfcl.multi_turn.miss_parameter_accuracy": 36.0, - "bfcl.multi_turn.long_context_accuracy": 50.5, - "bfcl.web_search.accuracy": 74.0, - "bfcl.web_search.base_accuracy": 74.0, - "bfcl.web_search.no_snippet_accuracy": 74.0, - "bfcl.memory.accuracy": 50.54, - "bfcl.memory.kv_accuracy": 43.87, - "bfcl.memory.vector_accuracy": 59.35, - "bfcl.memory.recursive_summarization_accuracy": 48.39, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 84.3, - "bfcl.format_sensitivity.max_delta": 13.0, - "bfcl.format_sensitivity.stddev": 2.88 - } - }, - { - "model_id": "xai/grok-4-1-fast-non-reasoning-fc", - "name": "Grok-4-1-fast-non-reasoning (FC)", - "developer": "xAI", - "scores": { - "bfcl.overall.rank": 12.0, - "bfcl.overall.overall_accuracy": 58.29, - "bfcl.overall.total_cost_usd": 16.27, - "bfcl.overall.latency_mean_s": 2.29, - "bfcl.overall.latency_std_s": 7.31, - "bfcl.overall.latency_p95_s": 5.34, - "bfcl.non_live.ast_accuracy": 88.13, - "bfcl.non_live.simple_ast_accuracy": 76.0, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 93.0, - "bfcl.non_live.parallel_multiple_ast_accuracy": 90.5, - "bfcl.live.live_accuracy": 77.94, - "bfcl.live.live_simple_ast_accuracy": 82.95, - "bfcl.live.live_multiple_ast_accuracy": 76.92, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 46.75, - "bfcl.multi_turn.base_accuracy": 58.0, - "bfcl.multi_turn.miss_function_accuracy": 39.5, - "bfcl.multi_turn.miss_parameter_accuracy": 37.5, - "bfcl.multi_turn.long_context_accuracy": 52.0, - "bfcl.web_search.accuracy": 75.0, - "bfcl.web_search.base_accuracy": 74.0, - "bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl.memory.accuracy": 26.24, - "bfcl.memory.kv_accuracy": 20.65, - "bfcl.memory.vector_accuracy": 20.0, - "bfcl.memory.recursive_summarization_accuracy": 38.06, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 74.09 - } - }, - { - "model_id": "xai/grok-4-1-fast-reasoning-fc", - "name": "Grok-4-1-fast-reasoning (FC)", - "developer": "xAI", - "scores": { - "bfcl.overall.rank": 5.0, - "bfcl.overall.overall_accuracy": 69.57, - "bfcl.overall.total_cost_usd": 17.26, - "bfcl.overall.latency_mean_s": 6.74, - "bfcl.overall.latency_std_s": 12.78, - "bfcl.overall.latency_p95_s": 17.57, - "bfcl.non_live.ast_accuracy": 88.27, - "bfcl.non_live.simple_ast_accuracy": 77.58, - "bfcl.non_live.multiple_ast_accuracy": 93.0, - "bfcl.non_live.parallel_ast_accuracy": 92.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 90.0, - "bfcl.live.live_accuracy": 78.46, - "bfcl.live.live_simple_ast_accuracy": 84.11, - "bfcl.live.live_multiple_ast_accuracy": 77.3, - "bfcl.live.live_parallel_ast_accuracy": 75.0, - "bfcl.live.live_parallel_multiple_ast_accuracy": 70.83, - "bfcl.multi_turn.accuracy": 58.87, - "bfcl.multi_turn.base_accuracy": 70.5, - "bfcl.multi_turn.miss_function_accuracy": 59.5, - "bfcl.multi_turn.miss_parameter_accuracy": 43.0, - "bfcl.multi_turn.long_context_accuracy": 62.5, - "bfcl.web_search.accuracy": 82.5, - "bfcl.web_search.base_accuracy": 82.0, - "bfcl.web_search.no_snippet_accuracy": 83.0, - "bfcl.memory.accuracy": 53.98, - "bfcl.memory.kv_accuracy": 41.29, - "bfcl.memory.vector_accuracy": 57.42, - "bfcl.memory.recursive_summarization_accuracy": 63.23, - "bfcl.relevance.relevance_detection_accuracy": 81.25, - "bfcl.relevance.irrelevance_detection_accuracy": 79.43 - } - }, - { - "model_id": "zhipu/glm-4-6-fc-thinking", - "name": "GLM-4.6 (FC thinking)", - "developer": "zhipu", - "scores": { - "bfcl.overall.rank": 4.0, - "bfcl.overall.overall_accuracy": 72.38, - "bfcl.overall.total_cost_usd": 4.64, - "bfcl.overall.latency_mean_s": 4.34, - "bfcl.overall.latency_std_s": 7.22, - "bfcl.overall.latency_p95_s": 13.5, - "bfcl.non_live.ast_accuracy": 87.56, - "bfcl.non_live.simple_ast_accuracy": 74.25, - "bfcl.non_live.multiple_ast_accuracy": 95.0, - "bfcl.non_live.parallel_ast_accuracy": 91.5, - "bfcl.non_live.parallel_multiple_ast_accuracy": 89.5, - "bfcl.live.live_accuracy": 80.9, - "bfcl.live.live_simple_ast_accuracy": 89.53, - "bfcl.live.live_multiple_ast_accuracy": 78.92, - "bfcl.live.live_parallel_ast_accuracy": 81.25, - "bfcl.live.live_parallel_multiple_ast_accuracy": 75.0, - "bfcl.multi_turn.accuracy": 68.0, - "bfcl.multi_turn.base_accuracy": 74.5, - "bfcl.multi_turn.miss_function_accuracy": 68.0, - "bfcl.multi_turn.miss_parameter_accuracy": 63.0, - "bfcl.multi_turn.long_context_accuracy": 66.5, - "bfcl.web_search.accuracy": 77.5, - "bfcl.web_search.base_accuracy": 79.0, - "bfcl.web_search.no_snippet_accuracy": 76.0, - "bfcl.memory.accuracy": 55.7, - "bfcl.memory.kv_accuracy": 43.87, - "bfcl.memory.vector_accuracy": 56.13, - "bfcl.memory.recursive_summarization_accuracy": 67.1, - "bfcl.relevance.relevance_detection_accuracy": 75.0, - "bfcl.relevance.irrelevance_detection_accuracy": 84.96 - } - } - ] -} \ No newline at end of file