{"row_count":32,"scenario_count":50,"schema_version":1,"stacks":[{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":null,"sha256":"sha256:44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8","source_id":"sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8","source_kind":"local_file"},"canonical_id":null,"corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"granite3.1-dense:8b","endpoint_id":"sha256-44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"unresolved","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-granite3.1-dense-8b.json","run_id":"20260720T191246Z-c356059e","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:1913a314978496fb28674ad2f5744f6eb8bc0197492546b1fdfc7972f8be40a1","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-calendar-followup.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:545b385d7ab07f631cc92bfa83566ce309fcc2ea4ec259bece16e70796cfdb8f","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-flight-booking.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:aa1e2bf5775f118227b5f860b5e8fa36aba3d5a3d5c6ce2878144ab1f27e4863","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-inventory-order.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:fa6ca1c8c3a7bb12954bd0a0fd313e35c1209e1fa1025b6cd7d56f4c2f44495e","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-parallel-followup.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:f89d292812558fbdbd1bee7c2bb1093c9148ecb55ffed8b72382e2d5afddc4b8","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-route.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:55681175a80aaba055d77080a9f55d9fda467d550dba4cb9c73572fcc1309670","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-ticket-update.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:3eaa3dfe396aaf37a5b8168e031120fe607fd583bbdee2c22ace75d6275d1667","evidence_path":"evidence/20260720T191246Z-c356059e/multi-turn-two-hop-chain.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:10d0fa4edf6c41a377a418a0dfe6a90f16b2f58f2884facdb491a7f7196fd83d","evidence_path":"evidence/20260720T191246Z-c356059e/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:241dac890d7f3589fe389e21a84e5c96c09bec62de51d80d3ed308a8425e43aa","evidence_path":"evidence/20260720T191246Z-c356059e/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:dd0e33a337d3a872078eecae2fe4cdfa940ca36899166c6a99969a255728e1fc","evidence_path":"evidence/20260720T191246Z-c356059e/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:b64f512331acdc475c652284a4c59e40cd33ebb6311f5f609a12fbbefca57e4f","evidence_path":"evidence/20260720T191246Z-c356059e/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:508d4a65284da8b0a2cb07c250bb1db21a2f7772bf692e4fc0849696a9d04b9a","evidence_path":"evidence/20260720T191246Z-c356059e/negative-long-argument.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:5e99397aefb45271e134707d131c6213527c4bc1ca8234184c14dd65d57be5bc","evidence_path":"evidence/20260720T191246Z-c356059e/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:eaa321153a548338bca8e39d1f405a9a3edbc46098db59fb4649a354311e4dc0","evidence_path":"evidence/20260720T191246Z-c356059e/negative-unicode-argument.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:fc391a12d072641cb5d07658880517a89b0e62457f93d5f1c776a1f3f5d2e0aa","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-city-time.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:56da12bd96ed41deb3f46ee1ebebe941c857576c02b82a4e3a044e9091c24bed","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-different-tools.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:bb0c46e91e7f3826f368d36f663fad8c48d421fcab4282b7d74c8eac57a9254a","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-three-cities.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:68e038f86d509bbb87f7d3ad7f463a5a1626d81bbb82e7018e1b0c0797b339ef","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:3d708bd902be73eeda78fede43d80e9b072a7bc9a0a608b515eab5d46013ee95","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-two-calculations.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:bd1e44ba53abab94ab781613a3e6dccb2d9057307f7cd4e4aa6c755a1dca9eff","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-two-multiple-args.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:b31d1f0fa5662f0d11b6d6ac1f16e50f27cba804de994b9f74c637d76bed6370","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-two-searches.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:09136e0e4306337d2111c6a70811704b332c49f8c954b38c01062d8345f452ac","evidence_path":"evidence/20260720T191246Z-c356059e/parallel-weather-and-time.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:807cbfb32a18dbab7898850cdcaf78b9d076a8200befe4b42b972d85141d301b","evidence_path":"evidence/20260720T191246Z-c356059e/single-array-tags.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:3ef57e29a732809b48fa2aff1fece610a70e318a93005e9d4a034140b592d620","evidence_path":"evidence/20260720T191246Z-c356059e/single-boolean-flag.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:1860b96669efccce2572b4eb8fdb305bb6477b50c632e3d6639b03c6b55c0299","evidence_path":"evidence/20260720T191246Z-c356059e/single-date-range.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:7c958d83842bfbbad18dc140d0e90004203e4b92d396c6a4bc6515cec5f24ac3","evidence_path":"evidence/20260720T191246Z-c356059e/single-decimal-price.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:c6d96c4fbdab7861bd6d0f7bc59f5127e27a70654982a76e8d6923bdccbe3eba","evidence_path":"evidence/20260720T191246Z-c356059e/single-empty-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:11d7048d214c745c4f54b1d4eb32d1045149857d1978b6434bedd39449d6a072","evidence_path":"evidence/20260720T191246Z-c356059e/single-enum-format.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:4c088e5787265f072d2e76adf590cfee0acbad2841aba94b70cb1ba5c35bcf3d","evidence_path":"evidence/20260720T191246Z-c356059e/single-integer-limit.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:175208eeae2474a2b25265b88b228b038bd16fce0c2402c44c41c96005d756e1","evidence_path":"evidence/20260720T191246Z-c356059e/single-nested-options.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:bec940a91a149f832b4eb3688768fe52764d81413671a1c302fc35f937d5cf25","evidence_path":"evidence/20260720T191246Z-c356059e/single-optional-omitted.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:cf7e808fb655512ce1e571dd73a0614201000dd3b112fd02627d3f4101fa86b2","evidence_path":"evidence/20260720T191246Z-c356059e/single-optional-present.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:a48fe06f54e446e380a2b420921546a7a25a0d9861a1a1b049f0cdd5c8dbdad6","evidence_path":"evidence/20260720T191246Z-c356059e/single-string-list.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:be320e9d802ad416a199454a153afc6977eaa60b67d6ce39f78cb59ed528ffa7","evidence_path":"evidence/20260720T191246Z-c356059e/single-weather.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:ca0d1be0ab5ed15825f1f829833231f2ed60695014a847f3a9c84f79daede86d","evidence_path":"evidence/20260720T191246Z-c356059e/single-zero-and-negative.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:b8cec126c21114d9cb7612932d96f52e001ef0450be2922adc32ffe9dff6b03b","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-array-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:3a7685fa39578c2f219ce160d7f8d96371f25e43ba42db6cd3b5a98d65464e39","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-boolean-argument.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:49a5970401aae6788908ea992b6fc8ccf5cdf8ec400f8018437a78d2bce514f3","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-empty-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:06ba5592e58cabbc9307f3a5f8b8347c22538004eecde41fe73f8f1c9836ca78","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-enum-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:10d943696eadad01e159829b00c87df21638ec4d28c204dec513631e5ad61e4e","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-nested-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:561a4f82cc52bd9b6e885fa6fa3d2f6d2b677336f6b00840e189fc89e2126465","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-parallel-three.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:221148b912cffac1b88753ee0e471887f091bc40e91201d7790223e675596e6c","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-parallel-two.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:73515b83d93033cd04dc3792a21c87da097222783b1c580765a465e8d6cb5ad9","evidence_path":"evidence/20260720T191246Z-c356059e/streaming-weather.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:55bd5605b114593a92c26921f3fc6019197123e83b161b0f319ab7a314524083","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-auto-call.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:e63fcbdcaa0eded7f40165c6f7ae575c51968695a77b6f534fc2f6f6f302e7fa","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:20efb34708f745cd361aecb7614d4f5c73f6d498c3733aa1f710adff01025cba","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-named.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:b254af567c1be328a674e48046f233d6f5416f9457e92e4c397a13c874623118","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-named-empty.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:6732e9ae051d31d11b03c58af8e127ac7fede1ec2049ce929e0135752b5fb612","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:41a3ecc337b6c201865f5a3ee9342634b1d95ee3ea3be964fe6eba0716ce7cc6","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-required.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:73addd415cbe0112f2d503a9f80c31e64447d5d57f79e189b2e65e3e85a04930","evidence_path":"evidence/20260720T191246Z-c356059e/tool-choice-required-empty.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":1,"timestamp":"2026-07-20T19:12:46Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q3_K_M","scheme":null},"revision":"bf5b95e96dac0462e2a09145ec66cae9a3f12067","sha256":null,"source_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"meta-llama/Meta-Llama-3.1-8B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"meta-llama/Meta-Llama-3.1-8B-Instruct","endpoint_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q3_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-meta-llama-3.1-8b-instruct-q3_k_m.json","run_id":"20260720T190335Z-549a61ea","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:bddf099204d09a2d4621f080c6b48483368c409e58b9127703d7718c861b962e","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:28ba83493929b65a2828409e3a27ce759e3d0d3fc1e3a54470eb7559870b4899","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:964f53a98c1d88d4f54f39d2aab1ba0845af0d380942f389ab355239dd4f06fa","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:310553bf135ab05275ab7a8cf3f4ebc3910c5d6ef31f03b15d5f3df7f938bb8d","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: expected number for 'first_value', got string","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:a28da094ff43cc02efb7cd89a150dc92aac0b0e8db3e1bebd63e95feeb05e800","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:fe145d21ff6cfa117ec1fc1c2640598bb115f63eb0e743c9bebecc98e64c0be9","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:6dfedf64e656fdf62643772ba4dbc0fb8c3578a4f9111da457c47d6c76e153dd","evidence_path":"evidence/20260720T190335Z-549a61ea/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:eaadf8ec2b60a008f48e08f7beb82e3f2f3e4b97c37ba362ea1817f30e94778f","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_weather'","id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:501fe3c6511e520bd96ef96f1ceb842fffcd9a9f27ecc298e0718176134d19c3","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'lookup_order'","id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:f8178f4acf51c51189391e0c15cf40f86f2ab9498761c54cebb2660bd58a346a","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:2c356c7937ab88bbab1c792e784b684079b90084760acde173e5e917ce5eab3c","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:99fe18db5d98bf87cd61dd1f9baf640fc785e82e78a398f84206a2b47d540254","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:5d69eeefc34783ab0eea7a68841fb513b99af024a349c029978186c88a4cfefd","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'send_email'","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:50dc755bef1371ef8948374f32a93d6d681f4d2e69bda5a78ee1fd5fcac3ce89","evidence_path":"evidence/20260720T190335Z-549a61ea/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:eb3a4be7f17e774a9d2c29440e65f1e870429a0a12cf09d7f33195691a2228d7","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:2bfd03c2f930e393f0f20fe1b158abac247d4cb0914866d6e394e164c8a4331e","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:8b0d5f18acd99c5615e0cabcda00f0a33e31acfe23a03f28ae9f54c175d68c32","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_weather'","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:315ee946f9a69ed211362f7854a7b9e6d718262837ffd1c0a621724a720baa29","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:de50018b3f29a42ab3a79eb2f72a4b892c4fc09c79773f57c64a3ed4916f822d","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'divide'","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:6a9742935ad392b45bcceebcc0ce1f9ee78a1ea78e937f564e5007e7c89595ad","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'reserve_car'","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:067c28e08faf68be3c9b245f2c8f468540f31384a2b1fdee4d06f284bd30d6ae","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'search_docs'","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:a32331d26dd50681196325a357a8abbce18f9f657f28ba2d0e038be59436d825","evidence_path":"evidence/20260720T190335Z-549a61ea/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":true,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:1797323660fa23fc1b6d63a343a80f8b8b823e484da25630b60cfcde91f1770c","evidence_path":"evidence/20260720T190335Z-549a61ea/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'tags', got string","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:dffa2192f9b0f9585e3de13b67a9d71639a7e9bf8dc3d86a851371bda7fcfee2","evidence_path":"evidence/20260720T190335Z-549a61ea/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":true,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:f133fe07cf430304c64cc2084f270d9124eef44f65ec315540f0d1b85e41f5f4","evidence_path":"evidence/20260720T190335Z-549a61ea/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:09a120e2983e6871afa20e65807f8ec1c3e04d6a278a6222c1c35f02778a37e6","evidence_path":"evidence/20260720T190335Z-549a61ea/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:4643b7265082cef578ac7505544c8998e4d486298c8ef2a7dd9df030fac43379","evidence_path":"evidence/20260720T190335Z-549a61ea/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:9c6e281dd39a13699df5b1e144aee3052dda46c0626a645e7290960900be9544","evidence_path":"evidence/20260720T190335Z-549a61ea/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:b5e66aa82ed98df9b5818274e5818a36a5776bca59efc219805e3bc7e8fd5cc6","evidence_path":"evidence/20260720T190335Z-549a61ea/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:0666068598aeae1db4cefd29cb2207dbb302301add69e251a754ac632a41150d","evidence_path":"evidence/20260720T190335Z-549a61ea/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:78a1c641d48877a04c0dd7d83bcda63f62b038cb172dbdc694a9157514d78f18","evidence_path":"evidence/20260720T190335Z-549a61ea/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got null","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:a6c3cb60ae152a8bd61eb27a08312bd90a235329148494a1e08ad30982bca7ea","evidence_path":"evidence/20260720T190335Z-549a61ea/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:a5a6a3a4745f7c064a08f71482e45385e3d0e52f88abf86b55db2c1e068be875","evidence_path":"evidence/20260720T190335Z-549a61ea/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:17c6fbec51c485ef47dc9dcc5c69d3761aaafb09183cb8dcd1c582d61ae5f3bf","evidence_path":"evidence/20260720T190335Z-549a61ea/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:c2c5dce7c78147c985d98420de85f084d42d88fd01d48adbf6ed9bd7d4fb465e","evidence_path":"evidence/20260720T190335Z-549a61ea/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:12d0cb6b6f60a743f5db4db35811ef671652bd95bfb380cb4ab4e9f16e805ed0","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:92c16ea87dec405fa60e72190a7dc280b1fce3ec4c0c7867c99f255e2b44ad3c","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:84b8bc2e262ac8eb7f4169af30682fe5ad76a8cf428908973aa9f64ec12bab7c","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:1d5294c86fdf7a9095f26ab9338dc6eb4b78af244da4b92408f90954d694ea6f","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:267f4b14306436b84e4c5c97b64a0742029ebf518a7dafb1205b6d9844595ae9","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:d8df391f3ef0561ec5b7a35a00cb0e366fdb7ff7b2cc6049bb6584c148edff52","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:34f40653135caf48f42d9f7e400a8723c1ba596cfb6e8f8346df018be763158c","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:037ddc08672a26c818b179b74b302c48207ec2b2d0b15bdd84664f284521cbd3","evidence_path":"evidence/20260720T190335Z-549a61ea/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:abe2de4695aab178bb1b6d91b31fda54974ad7cad5a2d0af96ce1c5e4ce1ed68","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:b7c579013a91028469db7043960cf665b0667ac23021e31d5eb3e16b313981c2","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_temperature'","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:f764fc312bc285be256e7884a85b2e575798c93e09b8f32ffc2c3c3c1658fb51","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:40d0c237dea06ab0b7b47be39ac5009ecf300953badaa2c9f2350a94f51b2591","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:f4b32f650471258da25d4ba80cabc8307ec5757c9aed0282a5ef18c9eec661e2","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:8df3682d869d29ae042a04cefea44500b73bce2e3cef85d797a48e5943ba290e","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:cb9c7480d15715be6c835a36000952ca975060fd848a924c127b74065b1572e2","evidence_path":"evidence/20260720T190335Z-549a61ea/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":2,"timestamp":"2026-07-20T19:03:35Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"bf5b95e96dac0462e2a09145ec66cae9a3f12067","sha256":null,"source_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"meta-llama/Meta-Llama-3.1-8B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"meta-llama/Meta-Llama-3.1-8B-Instruct","endpoint_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-meta-llama-3.1-8b-instruct-q4_k_m.json","run_id":"20260720T185910Z-fbefd8ac","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:08339d7cd2df4d19b24d82f75737edaef8bc91eac4497580d2059a189034e9e4","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:511c58b50a471dd2b5516afe1de62e39c70225bd90e7c737854fcec50b05e38e","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:56a876006439830c22c0fa35b3163544d427c310ee9bae05a7fc61e09d73b727","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:044574234e38b0cd7708d467b922f936ccddbe830542fa9805ff402664b870e0","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:b505f7d308b50736e974dfb6c65ea7361ee76dd81f31efdd625dd3146f53efdd","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:6f33b2e4098832188d5182edc45c1fe11bfaba4f02cce86218201401be38faa4","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:4cd1ebd336f2259ad405cabbcfc4200a869267ef54ff56154589f24e600804c0","evidence_path":"evidence/20260720T185910Z-fbefd8ac/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:e9c3d176616fb588eb3c18d2fe0c7968d7fc811d2f08238747549c23328a7c0a","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:4b95b982626dcaefc859bd0886485988759645298ebdeae11d4429d02bd2be73","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'lookup_order'","id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:afab8672ab2e56783133c58a6f91bf6bbf638d15c44fd1647b81ff80e2c48d27","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:c31e9ad01f248e422b2027f1ce3cdada8660f3d4156b64141ae8dbe095d2ffc9","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'set_temperature_unit'","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:07ea1c5e11c9ea039149538f2a91d02640ccfce4a4fa291f89635d7b6bc42e1b","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:7c206fb8ab930220575de41f980a4e9654d4eda1d01e44e62e7257f72de0894e","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:b1bd1aa4c12562b4a6273eeb6b399d4895eeffa2a724c3d615ecfcff5f6c1d1b","evidence_path":"evidence/20260720T185910Z-fbefd8ac/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:805eb6fff65ddfd7ee5218c64ebe456ebc08192300b96ea915d7dae36cfafaa3","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:ac5a9ad9578d0f8a217b1b4cd1642d2557e95cc82467550461cd4c497ecc05b4","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:da6d5ec9bea78e71116e31bfd14e2d5c9fe32400ed97902631e760ee862dca2e","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_weather'","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:5d3df10b6405740b1bb34d2ada9de1cbebd36223f08a8218db00ed44aedb1d67","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:90fdd7783cac3d4494e71eef95a48ce6a1cd9fd4998da11d87e70fb5fc8e455b","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'divide'","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:8cfb493534f469c552f84978109d0ecb25202ed7628cab24dd7b94ad342a149f","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'reserve_car'","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:bc13000c8e9fb3e00a4fb523d506bff1563045496620bf3cb2107f999d5a12da","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'search_docs'","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:d4650a12aa8597ea4a43f19cb7004209ee3d6c9866134488cf86f1f7a65f4562","evidence_path":"evidence/20260720T185910Z-fbefd8ac/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":true,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:b90f268bf6bbeb8f8aa0ea380d698ab1b98baeba6ae91332d08adb0430caeff6","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'tags', got string","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:548c8c23076e4a5c33527b18d9c5a90d178e55885d5e40bd3223a462dd10ffd1","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected boolean for 'enabled', got string","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:c20fc871a942dc91b6cdad68c9b9d54487fb0b6dc888842cfd33cc87f1560ea9","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:30f04130f7741522cabfa678992423b247ef62522fb74cc6e782d227d1c657b4","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:e3a2339ad16f8d4f3cc2d31044a568b8b6e6c1de024b430e51e2fd0d3b992571","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:9785103bee245848ad09c13877a6df60480e852725c9e071f580b9dc96ed2e50","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:75a7baa6bde27017d782921ec8fb463b3d1d57cd2e2b1b0beed5127972017e09","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:f80576637807be63679457285f3322be6c06da19c9ef2687785baa5adeb16a0c","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:b33b02bf72ca0b169bb97c66b5a23cecc165bb51462acdc5f8840eb87e5537ea","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:1fc70bea8803178f136f885187cd070156152cedda945d9d3de7b7b0ccbbbe6b","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:6eb66a73c942ae455053b6634567991294b46c2a867e896ad664101d2b920860","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'track_ids', got string","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:119b409cadec64eb6edb1a711a559b2104f3e6f24fd26a3df93e411b3dc6444a","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:0f7e833e77eb9766be092dc068a085564f7a83f672832f50442a95a6144086ed","evidence_path":"evidence/20260720T185910Z-fbefd8ac/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:48021f4a63609757e64eda40d44b2e679a3275718c0521cd29884ae451f95e5c","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'record_ids', got string","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:add221fd75bf6e24f113ef8a1bf095b990e29902fe15bb8b93f5e09d68e8c7b7","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected boolean for 'enabled', got string","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:1715bcb7223f0326cec1e80aef9b4ed20092a3dbe9ff31b3c9c05e6699cafc34","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:4e50c6ebbac583dcdae39817d07b9028db381084964a4cb7d0cd802b1a130bdc","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:32e3dd514a5ab6124ba70bd4683cf428f9afcc5cb6e3a47d1a67358282fc7b0c","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:7a1a5fea89908769b4361e2a9676222393c8e06da5a080d1fb94f54b9f379749","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:426dd8f84106f2359cb64c71c74e5c36f6181c2d890afaa5c1987a55239cae40","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:8a47bdbe80f1e072aeda65019503cf19eb1ed300ae2e9073158e49be13b0c93f","evidence_path":"evidence/20260720T185910Z-fbefd8ac/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:eb1770c51aee958ada7c4ff3a91592432edf899485cd6f37e5e65ca45581834e","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:74e575facd12a3c1ba75bce61ebc44ee879e6b1e3b1d612a273e4e5b98313010","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_temperature'","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:b642347db85584e0036d48743dc4b1e5f53843e2a00dcb78783ce02b2458033d","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:69371f0efe906fc22b010eb40c199546fd44aad0db67f07a2f0bb52bc06dc0ee","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:f4b59da0b4c2d0eb50c67643c81257cd49e0070edb19b3f7838e234ee67198eb","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:e7d573cb789f0a24f2775d74d7b51c9a089ebc8bc217001c1511e3568336f95d","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:8892187a6d259d8cab2caa7c73583c11ae87a9d8f9880e9fef71dba9dcdbba26","evidence_path":"evidence/20260720T185910Z-fbefd8ac/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":3,"timestamp":"2026-07-20T18:59:10Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q8_0","scheme":null},"revision":"bf5b95e96dac0462e2a09145ec66cae9a3f12067","sha256":null,"source_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"meta-llama/Meta-Llama-3.1-8B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"meta-llama/Meta-Llama-3.1-8B-Instruct","endpoint_id":"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q8_0","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-meta-llama-3.1-8b-instruct-q8_0.json","run_id":"20260720T185633Z-b808ab3d","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:ff93d49ac10032673a408942eaa462e4fbcfdebc835a7b6824cb69a98345a610","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:9794dd1d3cc9a1c85d792d7da0ed8b481d085e2cfda8f34bbd9136cc242ff9ea","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:eb30f5be7c7b61592b39237dd61fa0912a4055979162e1696ced9f348f4a77b4","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:7f6edd8103d1f4bd280066c64c37b894aa3dc4a42b0b8c3b1146516e8f4cebdb","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:6880df65d411b9e2f34eb15ef6a99adbd2dd0b102fcd595731f694403f4011e7","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:172924e3fe372e989966f1aa73c25ea1791a248848a06cac474e2ef1d7804505","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":true,"status":"error"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:d8415caccae99c12e03b18c4e1a1f404b6d5b970f158a5945956b4e0d1223f42","evidence_path":"evidence/20260720T185633Z-b808ab3d/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 500 Internal Server Error after retry","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:0541ec3d82d8be585b957950f35321dd5c0aceae6cd2f25b93627a93c9768609","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":true,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:238d88bae5999add1f29fa2d2252eb667c9eb1cd2acd3b5186de803e48c66665","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'lookup_order'","id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:f1eebe782806b1df235b6aae9be8217c360ce2c5061283808eda8bbde4e06809","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:25ff8ba63c9f8ff2cc18317d8e322fa525817f9b0fb7a2fd70c3fa7b7aa75abc","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'set_temperature_unit'","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:135b0f4ca6f82455c1da6dffea54864ecd230e249dce9dcc824d7919a16ee791","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:66003232c09ee4cd854dbd9fef4507f21f51e16265c13aabdecf03204c1bff2f","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'send_email'","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:93b689efdc59b2f2db38dc4f77d1183a65061669e94db64d03f4de2dc27fb6c1","evidence_path":"evidence/20260720T185633Z-b808ab3d/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:28f300f34c08d845bd63a5881c1a888b26a64eaf02a6d84dcae0de2a16c23609","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:b29fc076c908cad96d216b11d23e521d59731e088909f414f5c193f012823131","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:c8543af6591daf69f9048c94318c23eeebaafcc596622a13f18da81319aaff8c","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_weather'","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:f37a8e1dfc60fd45f77ad25c9248539055e75e510a6472df6d008d2056426f69","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:ec6fd185c796f1c495254bfac859d3a169578d389579dd4402baa7d50bcfd78f","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'divide'","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:f2a4ba65dd18e27115323cad9fba56bd660595366429cd91eafd2a0ce28ff5f7","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'reserve_car'","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:1c72268f90b3f2e6f355a49f4440b17df65b9dc93319a91e18bdbeec11ed727e","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'search_docs'","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:f8afa91d329e2ed460f167eed3b5ad3d199f2400c90ff1b9f0eca260401e9c09","evidence_path":"evidence/20260720T185633Z-b808ab3d/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 500 Internal Server Error after retry","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":true,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:0976e0312d17d50ee8470f24be236e6cac97b62f802adf73c2abe17e1b8ca5bd","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'tags', got string","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:7d29e29d238d581e3afe4220175d3286ebc6aa1ccd9f5473c940b07c37885949","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected boolean for 'enabled', got string","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:c6f57de5283100858b8073b316f32bce6576445bd8b3a372c0303f0c4397e85e","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:9d0676efaf764a878d447ed1546774876aee6fd8517ccacdc6258cde263656d1","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:0a683c136242f23ef4a0fc31012af4a39be867eebdba2c0f356e991d8cfee703","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:a255ac1cb4f11328e94c84a419f6f98c3f201bf3308af5e1a1a1d10e5e423acc","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:d7d13497476ff5538413e494a47c227d3a9d0bbb9b0d4d5231f46ca4875033bd","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:707aa5b8ef47d678380f165e88df192eeb72892965cd24e81d2f9239daf24157","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:33b9ae6763c06c5a8cfa79801460ec986b1a5e30b1ea20694601180e24b2096c","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:800d0f3edefb7337a79702d01bbb639a237d2b7e8f8c284ec156aab7b4cc669b","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got string","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:dc3ba231af23af67d0798d6de88c7a6dccccd05c22ea0d65a0e0b59d46ae2a65","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'track_ids', got string","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:2d496d3fbbf13a9fd2480ae00a4a9ed9f9c286eb37eb693cff0dee239f71128e","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:d83f610540cf5808c2d8e36ac3cce51e4035458845454499fb066b3f375e7f7e","evidence_path":"evidence/20260720T185633Z-b808ab3d/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:720f9dbb6cb7448fbe3c91f30135503dd0b7fb03752811f9c028a4dbfafa3fbf","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected array for 'record_ids', got string","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:2142e93b108450356ed66eae6c114221ba9474029a8db0622f5e3d681a4951a1","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected boolean for 'enabled', got string","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:f2cfed2ef0a3a09905021ef8b60a78c4b673d6907f06bf195a63ef75e63daa5b","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:ad4069a96a8775d552654c7c8fd5dfe424e988666801d65d53f69f0908e93bf4","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:53fe20d10b87db39e4b2b9a4d80704af7e761efbd7a8936833da08b3c919ac91","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:ea5fe92a60c3fbe230738135c02ff0c56e6ba493dcd3896930d088e210fa1d3e","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:ac87cc09317f008c2ed25673c0f641d5ee4c4d7728281fc25d923a3676bb025e","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:2dafb361aad4b833adef1167064070e5999e3c566d17216633fe81572b9387e0","evidence_path":"evidence/20260720T185633Z-b808ab3d/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:5d0bd3143aa82fa82e0da3ce4dbaed7bf877e3f257882e4013e3dc45eb8f7a4f","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:adb882ead337681efc015707d4cae95fa01745de9827ece3e47a852f72dd97a7","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_temperature'","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:6e2208ab1f66f0c9a3035c11d7d630a618abcc8dbf292d5f301611ed90dff94c","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:f441631e2db836c584a559d78951e80d09940a8dc5c206cd3ef3e8c2540d13a2","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:e60e8313e7b65e164bd9b05105c3bd548f3838865495098b6e6accc4e4cd98a8","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:ae8c850b654f204d6140140e7521af00b7d21410d4a9b148cc7c694987dd76ea","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:dea47440a97a4c698c99870965951ca9d06fe8e0a024eec6ee9916a1e89e55e7","evidence_path":"evidence/20260720T185633Z-b808ab3d/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":4,"timestamp":"2026-07-20T18:56:33Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":null,"sha256":"sha256:3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db","source_id":"sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db","source_kind":"local_file"},"canonical_id":null,"corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"phi4-mini:latest","endpoint_id":"sha256-3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"unresolved","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-phi4-mini.json","run_id":"20260720T191346Z-7b026ba7","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:4f309572f38870495b3a41fdb4eb625ecf64c93ca16cd1a431ef421656ec2f63","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:a478411e30604ba5437c8e3cf36ab75a2e4d9c365d014e56e51bd62dcab46e02","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:b9f1ccb94b0c06eb1ae55a5e86a2a76e3197b73cc393a3cc657adf1d8599f42a","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:036e08e40cdc11f25bfafb1fea2c1216738ffe8c17ea89bae7d0d643618391b0","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:943cae6204a9293092e44c02c4b1cd0d198b5d44fd20073718e61665866e991e","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:05ea2cb9510e8667d801374be13768606fcbcaf481d9075e976c09346f8e3f7c","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:44a75e8370c117a00d99537a8c19de1a397523fe9cecdc6a702d3ee667a01b40","evidence_path":"evidence/20260720T191346Z-7b026ba7/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:a982531a346b9ad1b85daeb450eba60c2f98c19722a9b0e5aac64ca19c93d272","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:0116a018df2a6d6eb6b348effa3ddd53d175b34a8320a7cc8295c489bbb0f59f","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:5614a85dc399a3b578d187118f9b8de7124b66b28ec0def37fb9ba79702378b9","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:dcd9820759826e5eaf417ad7d9665a2e4207407960ad9254c0c035c3fec5516e","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:c410a44fa4ec0823eaddc5839d8c8b8e020efe8185ac042244d73528c884ea0f","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:e2cdd72fa47bc0e34f452a6d16d81cc8f13251d940ba4a32bc57669d2d56f4dd","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:16173f856a270c9bd7cfe33d735d6efda84101e28ca03ea26f6f72473960f14d","evidence_path":"evidence/20260720T191346Z-7b026ba7/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:2e2f7fc995df4dac1148dc89ea2476dddfc76c2f094bd0240b215b3576672f81","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:cab8a53cd573c5dbd06f50dfe5f13401d4bcc53f7f1a362874d185014773fc0f","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:8fbad3dea2d9968e8916f3ce6ee0a7f707f4ffc78e1059b66f2cfcabad121cb4","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:2a6d3615f95ea7d8995a5b75e06b9f4570db27d1d9836e059080defcab33ec5b","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:1ab30f6937f57fe5d87f75d13a74fdeb351e8e29f3d359c82d351b265736d01b","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:5067a2de7392b5910670ee02fe12c6c3d6cedbc58379ee66646897de7595c35a","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:c4eaaee58c4cfc7097cf0e3406f38060b497fe8de6d25ad4cd4e66ca13d51828","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:28417466fc3cf8ce650a21341fd58825bd312db363cc373fe27e3324e78c4eb0","evidence_path":"evidence/20260720T191346Z-7b026ba7/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:35963badda79eb86d29ed6bf95679484be2ade3c2e4c91f3b325a79c91956976","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:e8d994e885bdf158608ac0c04a0e756b928a879725fd57d6008eb901bbc7a278","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:faff2ad0591becce3ec57d160342bcdbf9730cae3123aece4d4417758316a00f","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:cd4349411a6c3ffdd3a7d61de38ba3ffd7c2a8d1d0c2832932e1c42640040bdd","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:2bed2666e328be6584a3849c4f73bc9bf9c5146dae8965032f0aaf5de75c4c1c","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:a7db77448749f3574d040115f79e77f0e3922c16e57074ec0c8005fa200084d5","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:6ee15a4825ea01350ec6680f196c3c0f9c413e98d1151f1556a84f7eb428b834","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:dbf31bc798abc9799e5ab33b3cba4a1465548d1c250714d3ed8c457cd31ebb46","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:9d15f6b751f8672459b500a3e8d5ad623f1899bae442bd567071914fc69f9798","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:0d7c441c6f593b2a90b34828b7c65193a83911c3afad7c9787cccb47e00f97ab","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:007eefbdfcbde55b9b4f4a148eaba4cb1347bfbf8cd75ec5b91c1948554835c2","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:30d2a70c7e280bbe72b97709f5ee12d853028f549fb39b74992b1a4c59fcf531","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:908f93d9e113962bc5ada92656f87b52afbe82d95b449786fc691f7759d5fbf9","evidence_path":"evidence/20260720T191346Z-7b026ba7/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:7581f6a26968223141ec401c088c1916120fe6db19d778f87bd3c73c06bcfe1d","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:8765f234d378d332abd6cfbde251405696da3d8ecfefb2153d5496d298ae1591","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:2551e07068b3ad2acbd4a4d1c78ab4f79cfed7f5a233a45ad5786cfb5152f3a2","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:f56237dd5b79bda97d5d655158fcf8ebfd6c11d5990659c9e685788d2edaf93c","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:85079e5e197b27cb108d1f14f8a380ace86055ee0cf5cdd92c9cfd64d139886e","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:6ee36093f19e396995482f54e39d8b106e0ade7c3fadc20082de6992edcece12","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:d42c81d517ba3de33b79b1cea4427a058e35f2e13bdab72e557d0476b6b670cb","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:3dd41b595d2ef4b1b6d6f28ed1d30f6b86d2ea31a0e55278dc8672cec176c54c","evidence_path":"evidence/20260720T191346Z-7b026ba7/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:10aea5ff8ec8a102992030d2cf6cd3fa96f2d682195b43a1f2427807fe748876","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:c241f27b4672edf2306c786d5255f9c84beceffc037911a5864ee55b1effeaa0","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:61d90021fc59993fe7ca44024049e6252ea9366a89a379df330a3aecda6f0481","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:53227ad2370991a1ad06603989bf5f04fe006fb101d993926c1cefb7169b7e3a","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:624645bc4d3c6f3734b74c9b6056c55500b603804025c18e3b9cd4f8ae400dee","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:b1c684b8e3f97b23122345feef0b0228a7e129a67b37cc17ca2a5fe1a90231c3","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:ae31f68271c4ee679f323725605dfef2d935c13488b7d595ebc8e311e5056e3a","evidence_path":"evidence/20260720T191346Z-7b026ba7/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":5,"timestamp":"2026-07-20T19:13:46Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q3_K_M","scheme":null},"revision":"91cad51170dc346986eccefdc2dd33a9da36ead9","sha256":null,"source_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-1.5B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-1.5B-Instruct","endpoint_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q3_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-1.5b-instruct-q3_k_m.json","run_id":"20260720T191038Z-2737a9ed","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:cc62449e85b03e68d9b9fc0057e9d79f323ab309e28cb826033df16fe392de28","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:62c1e0a059775db4522e142e069cb98de99eecb0bc44dfd6ea7bd4c58390ea72","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:5ca4c046c2686cf3ddedbe7d6932d910a6b41f72c2e22f26952baef9dabd863d","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:933d9f32626af52fc4c39aebcb871285984b65f3e25ca1680162a8bd701eb174","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:a860905a1ccfdf65c48345583734fcf7cf11e59f72c3a7c558e2196e17b6c22c","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:76df4629c9d6b9998ee9f5776c5a4dc4121b98828a70f63474423555babe26b1","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'find_ticket', got 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:bd648cdf91fe9a3bc28227d01521a5f97b0be3e62903d4f5ccfd7d47120ba912","evidence_path":"evidence/20260720T191038Z-2737a9ed/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'find_user', got 'archive_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:af3e653fb5273bffc2fe7dad488e36a6e4fd7487a5bd7c580805e5ef5e6caf95","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:7a357fbb21bf836ea830e5ba831bdcb7eb29ae24e50e4883ff87870ed606e2cb","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:7b5f651f1a6b8eb9ba12c9d1a943204dea7236c97f626e60679ad46ff350d622","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:0c93d95215adc7f048ffe5eb735f890213a0e54ed4a5f8e96e1aae7424a42fb5","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:f0b25d55c8cf0d73ee1f2e5059847287df0f61560d406710b85985013e56f32a","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefgijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:45395bb892f907542f260262f3f0198e95106d22e8c4884fefeec224f887e434","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:84478dcd9b7b51420d915f9ae5502bc0a4db1eedc3675a5c2ac2f5e241038dfb","evidence_path":"evidence/20260720T191038Z-2737a9ed/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:942fc518a1cc7cd7576c4d9070038228a48615ba384dc79f0328fed4d7a84f27","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:7fca93b2ab3a4b09ec0ee177eef21446581fdc247796bcf71ee0e96cb9841c24","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:877c75c5958189dbea3539ddf1eb490cc1b06fc75d18a73cb409288d4e06f238","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:073309fe65047a20f7f477682490eafe2d9efb6d2ec367fa98b0d740ff153e37","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:738f13b47cacd04fa38bd53ebc8cae4f5e639568b3a6c16c86ac8ab7c4b66b81","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:82b20292b025704d2986e2639f34d2e5e841b210653693f37fcd9e85a2472bd8","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:00def3b34d697bf5f57e5130c0a874f87deab36b11f5b0b06a82510b5cecda18","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'search_docs'","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:a03f7dc00d77aaff2f4ef6cc589428534a6d106a3018446b6278d79f11b724eb","evidence_path":"evidence/20260720T191038Z-2737a9ed/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:decdcb3e309e3a019c7b382113a41904ea7e3100f30e0422ea1339c2b5daf3bc","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:fb893c4f18c290cfd39c3d603d75f8871ea185b0091ee070a60959bab8bd4aac","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:9fa7be689c8a16048e0c0956f19fa964307f1d681cd372d40a9913a20adb9cac","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:97f90ad5869c433bc8798cc9d728c32d734128612310682b50e5b4c2f3f3b6a3","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:22b94b8f567a63c8bed571ef65d81f76b8fb1c544e81b768a2038f93cf738e60","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:8dbd2639fe155323a28f14a57d92641ece5375d03086381876071c7aa680cb92","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:96888d9878cab89101b344e8a89e68cb42ba0dc70b063273247ac165b175d8ef","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:a6720b4cc8be40e1421c0311102800e2a503ee531fe6bc264c9e785ac33a9410","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:2810272f8015bda32b08ebdebdf901ce8658b8d79875799d85c591be71e05bae","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:077e3d6884c4b869a74062f612b0a03b514696d4c318620aa6640872604cac8c","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:0440fe8d374dbae153703bae96faee68fb5f4747c2968d1db17d5e745dda7de2","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:cef71167224ba4afbcc4eeec4ff447dd3aebc682f7d60906fd78d71cf7a5c72d","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:6445b5b8acf581341a86012a2e75d33a3169e3f7e56cb0c551e926ddc4654940","evidence_path":"evidence/20260720T191038Z-2737a9ed/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:e7be6da0e97d20a01e867a73c3bd6bec776bc1be5bea79482a94113ff576dabb","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:86c312d36f4d55d88c64f8eb1db8d5b969eab8896b992708166b102b1f900682","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:4ca91fbba4cba2d976864ebab18590b37641cb675e83cd0c0913c5f36e43fdae","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:09bcd993d22c63be741f174cc8abe838fb1b60c01d838594b2d9b6398ad5072d","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:7dbad1a74b46e25393225457b0a9a7ae06399781afaf9ccdc6a7d4cb0d08eeb2","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:7e4a56bd36f59223df9c8ec7ded61fba6d844d8323b2e72907a0940ae39edee4","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:66e29cea62597b25fcb4caaa068352dbfc84aad1b7f8c0b224fdec4919e18709","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:6d270aa6cba0227d50330f444ea771388d17aac1c469500564ad4317f57c6ea4","evidence_path":"evidence/20260720T191038Z-2737a9ed/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:5864ebc8c364311af59c69c487a113cada8571ba0fd670db06f2c399d6854fff","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:4c6a464fe5dee9b0e96b1c17d6c0a7bcf3928ce4346d5f312608cf7a54807330","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:4c4b9e6d7132950c958d42abc6cdd64d454312fe0aea6c49701b7daf04153f4e","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:37da18c7574407a1ed29cd4573da5fb5d2f5b1145cdb0c2a74f57f7c0d535782","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:8d46184649f297bf578df890d2f76a213d8d071f8a3c177c53c9287af88680c9","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:89b56c7129021f7a330bada7a764e61c783fec7aeeaf6a26fd9c903f561cc5df","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:cef91770dd44d50ff2c00360f61c0be17114e6b2c4a8d74aebe2c33652194ddc","evidence_path":"evidence/20260720T191038Z-2737a9ed/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":6,"timestamp":"2026-07-20T19:10:38Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"91cad51170dc346986eccefdc2dd33a9da36ead9","sha256":null,"source_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-1.5B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-1.5B-Instruct","endpoint_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-1.5b-instruct-q4_k_m.json","run_id":"20260720T190843Z-70268904","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:8e4c5039e025c129e6eb3451cb9675a07b159c98e510b3666579238b012bbd45","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:d0886e146fb2310ed81fd156ccc277418ec8cfff4bca920e246eb9eb48930f1b","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:2114851a1c9d5e382707f6ada3ca993ec10820532a3de9e7ca8915f4aa1de1a7","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_order'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:eaef521ee57dc7fcc227bf688c0f3c0debcdd45f0275e6cf39fb5dcddba698a1","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:afdc96081a5bd1373c558f726ac895bce5e89e487bec5eaf7e8e86a90b674094","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:c0d885c61ef98f5a5be0f8ac8ec68023a3f7d206d02458724eed7095c6ab58cc","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:f6e1e17abbbbcd92bc0d8e463346792f188e7d5d962ef04178a5dd6eea71bedd","evidence_path":"evidence/20260720T190843Z-70268904/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'archive_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:7c7c8b4e3e251d1272527fafe0a61de29ccc4c680697645735e31d043a3f8a7e","evidence_path":"evidence/20260720T190843Z-70268904/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:dcd9f7c71d4347a73dc4c843ce649d626240472833be0f076478960b83a0f19e","evidence_path":"evidence/20260720T190843Z-70268904/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:19a03b41a16ffd8bfb3d731d448db5a371ac5cab0e56adc206de1df7d80241d2","evidence_path":"evidence/20260720T190843Z-70268904/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:ede6a161b867ba8f028a7f008a3743a762ebeebfa7faf35ccaed02dac7506a90","evidence_path":"evidence/20260720T190843Z-70268904/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:98c9f5dd4061c6d646f05e056aa67c653ffbde7c98f322cfbc8b4b8005beda21","evidence_path":"evidence/20260720T190843Z-70268904/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:73c520f1ffb260185d2c25813060ad0aa4b756d0f270580276cf2516e57cc015","evidence_path":"evidence/20260720T190843Z-70268904/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:fd6a6a55d27f3bbaa8148b9dfe2bc4aebbcf9fc3a78c3478fbd3da56ed73670d","evidence_path":"evidence/20260720T190843Z-70268904/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:a60da2ea55254279313d0f68da6d8ea55864e01f52d94142edb620f9beb6716f","evidence_path":"evidence/20260720T190843Z-70268904/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:ec584a32c9b39c5c0b5bc54386cb85557695630c1bbcd24305bdd93064eaa524","evidence_path":"evidence/20260720T190843Z-70268904/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:c6e1f330d314adfbd199c4f2eaba210b29978a6cf6ac9deb48d87e423d75fde9","evidence_path":"evidence/20260720T190843Z-70268904/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:9c20a9dd72d6c9415cd1a628d8efa925eef6479e33aec74c6f1771764e8b5c15","evidence_path":"evidence/20260720T190843Z-70268904/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:3c11836ae8966b92d466f8b8b0882fa126f172bb22fdbdd218c8a9d0f87365f6","evidence_path":"evidence/20260720T190843Z-70268904/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:898d08b982d364410e3409179fb1566d2406c195393f5be9481d09d4ca6fe188","evidence_path":"evidence/20260720T190843Z-70268904/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:20c54e2c4ae28ebfabd413f68aecb296661292ce720f059554e69e2d72c981e8","evidence_path":"evidence/20260720T190843Z-70268904/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:fed45443e626d7b99de37adfa196fcbcbf0077826dfeafbd329e7df98aef24ef","evidence_path":"evidence/20260720T190843Z-70268904/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:3ef4ab9ffd04896536775786c1f6648549f3248400f5c53609d02d8b95b8a172","evidence_path":"evidence/20260720T190843Z-70268904/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:109ed3b246b582933a5c89555de9541abc094636967d6c57c0cdf4367e24cfe9","evidence_path":"evidence/20260720T190843Z-70268904/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:26b8a04406f0e75384cfaef3ea409c5690764be6fb3e0bb93237d6f80aeb485b","evidence_path":"evidence/20260720T190843Z-70268904/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:f203ffc449c793c1d1a3bec77986aee8fb7db46d006c183a0e3662a0bfc4e5dd","evidence_path":"evidence/20260720T190843Z-70268904/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:537834acfa20a715da7682d6509204c0317aff963533ecc644d5598dec55f47e","evidence_path":"evidence/20260720T190843Z-70268904/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:8bb4d61621cda76e9367ed237dce8751f8a32abcd79ed016fa11790f1ae1a101","evidence_path":"evidence/20260720T190843Z-70268904/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:6d5f992644a2e774fd54b15801e3cd7cb7238bc832b253807ae7508e26b8e7a6","evidence_path":"evidence/20260720T190843Z-70268904/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:af22e56a9ae4ab8fb03f0b718514356f59761e19733659f0af79245408640234","evidence_path":"evidence/20260720T190843Z-70268904/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:9cc0059edd2e5a5b5f6953936228d085da7cfdbe751f450f8b3eced19489cc9f","evidence_path":"evidence/20260720T190843Z-70268904/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:6981b5111567631d55e59cc39b8d0740abbdadd10108d472bd91c2d03317baf0","evidence_path":"evidence/20260720T190843Z-70268904/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:cce21e416631b7b7eff6822cd8764f660f050271ab964bb22689462cfc5e6a2b","evidence_path":"evidence/20260720T190843Z-70268904/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:2bc22e759258610cac627ed062329c317aa8e52d54617bf533b0556192f84e4b","evidence_path":"evidence/20260720T190843Z-70268904/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:cb7b3c93c646cf433b44c30e6608d77ef25cd0d9ae4fc2e6817ebcaf016ace2f","evidence_path":"evidence/20260720T190843Z-70268904/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:ba58db100954df5a9e7351448f877cee2f8f61c3445735fdfd938443873d7a05","evidence_path":"evidence/20260720T190843Z-70268904/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:69352861672435bd0408a6c07e8d27aa9e30eb8667f0e008854381e80b4bb645","evidence_path":"evidence/20260720T190843Z-70268904/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:3afd39ef1674a575c7f0143aa2480ba7a88857358b4591dc3a5ee72fc7bbaf58","evidence_path":"evidence/20260720T190843Z-70268904/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:7541e75efbb8f9ac62d693dad377107ea7811683fe3b374bd2d5d342a92ff086","evidence_path":"evidence/20260720T190843Z-70268904/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:103299ef9b76a11c650ec7e545c6b16e83eda4e6bf21cbd701b0f264f439ccab","evidence_path":"evidence/20260720T190843Z-70268904/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:91ba36fe31dbaca24f883fb2a445723e94976a93972f147a9433bea94f3de706","evidence_path":"evidence/20260720T190843Z-70268904/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:8e36a6c2ed44895ae93c98e631d96986a9b5bb75b01a969e4936a2ff58596451","evidence_path":"evidence/20260720T190843Z-70268904/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:af523883f0109305bdb2559deae24e654944a3354821fd7eb9b00a5e63ab7f0c","evidence_path":"evidence/20260720T190843Z-70268904/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:1f53bfdf2d86ee52c6093aa35b25e08f6a5e10dd17dea2fb562b64b295ce402b","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:1ec49ab16a35a8096326d72b7891c6cf12f75cb6eaad534e6239bb19a9afd71d","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_temperature'","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:22149d9aec5e570650bdb0250647a0c13f31414e6a9eb1471658ac980d1c997a","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:e86809ed96d2abfcfea7ec8664b4b9771673366e0de833af46c5176204fb670c","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:be98817a3077d04237f3cc284cf97c950001ec544a37dd320b975e6d1a10c9ed","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:4451387dceec6e41e335114f272b427189005ad2b8553dc0502f794a2eb27ca6","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:c1276d65654e3ded2bb569d793efea671843301e8e543fa4e057ef3e404fe86e","evidence_path":"evidence/20260720T190843Z-70268904/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":7,"timestamp":"2026-07-20T19:08:43Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q8_0","scheme":null},"revision":"91cad51170dc346986eccefdc2dd33a9da36ead9","sha256":null,"source_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-1.5B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-1.5B-Instruct","endpoint_id":"Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q8_0","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-1.5b-instruct-q8_0.json","run_id":"20260720T190726Z-21133f9c","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:55873e40214e79059aae807241849548403e123d64f6631cf0157ee19b8cb445","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:1d71d6f470b8aa166106bb8322f9b7adeaeac961ab8f1d7ae4ae5b9f72e04776","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:c0a621620745ab798208b6805d7fa40539ac8c3d5445c35403e2c4b038d4ff4f","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_order'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:a4aeff6a49e0618d49d3e61464fd6379fc201727a2e1ef9530d628516ca90151","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:59541c15196c7d6906be5ec4752e6c9b7e725707b8bfeef311a9c1133e7fa56b","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:56b00857f475505831b07b8f3ddba39722e08b733cf195725b7a7e831061e987","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:c86320b24f0f23cf8db951cc42877532c16ee80508286a2a8e547483767034d4","evidence_path":"evidence/20260720T190726Z-21133f9c/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:d8e493e9eeacd7585e7255e828faa08c08d658aa6b461392cde51b5f4a902e75","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:83627adb3effab2f5e9eb1f2e20a031d8dc1875b3d50c7f6a15f605a5113fbfc","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:0c1417fbe01667643f99aa65d42f1b32ab9cf8ad4bf97918f18d507cb5247d72","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:58676ebd4180a389ca578da0620a9593ed28b33bd38cc92ef36f858db711cfe3","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:708e04d581a3da82fe9a0b5fcbe74c2386fbc43aa0d8e7ce5ac2584a00f53c91","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:e5e0886a199917560cca3c0fde1d5d17c64f809bb2fa8cb52f6a32d998c44147","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:d1f4686d7b4efa416254b3f523b6c8eaf7a399666b32a3b6bf783aa8793ebd81","evidence_path":"evidence/20260720T190726Z-21133f9c/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:1b5fae6eff077cf8d80c10cbb0709d236bfcbd97db5a191e8199ac8f36230521","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:1ab35b6f274d2ba65aa66e2dbac803705725ae5bc372dab7da016fb97b786135","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:f73e49ce66f02cc14726d2cfa565210af859fbd15236493b33d5c91c2ce47831","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:7226e8af5d717b62b55b9e97d7cb9024770ce0d12719f2f38ecf7d4053cbbc03","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:7c7984ddaa93fe95118004a1e5033afcb893023c5a931bee644af05d2f617883","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:6d424db4208c5bb845cb69e960efa07c55b61255a3e042dab43ec826ef088c76","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:6c3722bba0fee2ce5948bee6fa433052d2f6e4e072151d6b967955c32e1cf7bf","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:eb153df465e0f5552555b0efe242b069dbaab86e62a4694225efc68903571280","evidence_path":"evidence/20260720T190726Z-21133f9c/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:bce24d39baeddd4a9f31d85b331c6815e110923c3154c2af51268f0331cc758e","evidence_path":"evidence/20260720T190726Z-21133f9c/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:fb4c90b4e7d4a9b917e2eb6cb5fbc3f626e31812f6d0176be3b2289ae81a4e07","evidence_path":"evidence/20260720T190726Z-21133f9c/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:64ab72c011bbdc4984e0f7525b1f898b9f337c272078c9f829b110bfa9a73db1","evidence_path":"evidence/20260720T190726Z-21133f9c/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:759262820d02b0771e63f65380aa0551afb55411accfe6ccec9faefaad011005","evidence_path":"evidence/20260720T190726Z-21133f9c/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:3fb05a5c6524290ca48ad0ec75540e93272bc1aa2ba33c5ee9c502c8bc8bf910","evidence_path":"evidence/20260720T190726Z-21133f9c/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:8ea7d43ace823e2b6635763e5867f11546c4c34ef5775029fdf95288e7482d87","evidence_path":"evidence/20260720T190726Z-21133f9c/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:60f1073662e65f5b28c4382704fe02afc8ee012d3ece2aeb06449dd6d505e3e9","evidence_path":"evidence/20260720T190726Z-21133f9c/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:d4d461586bd67f9f91c72846108dc5510db0afa5d356695e476b5d325ba325e9","evidence_path":"evidence/20260720T190726Z-21133f9c/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:5db774d6348447dfd7aa81a5640ab444d4f8bf929f751161853cab7891fd6565","evidence_path":"evidence/20260720T190726Z-21133f9c/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:447a925eff11ff7e3fca4514bddcfd2e519642a4fea1ac8f0c737b188a3b1d6c","evidence_path":"evidence/20260720T190726Z-21133f9c/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:262a710d83650d0d768486c80f96fc205b66fcc6ca2e3323110587e2cb526e6f","evidence_path":"evidence/20260720T190726Z-21133f9c/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:0d10966ba1ca7e76d0845a2e130598845de99e751d8243cf6c925275b539557d","evidence_path":"evidence/20260720T190726Z-21133f9c/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:93890686107051915b6500f8b27d443ba312a584c4d5c455316dba115a2f69a9","evidence_path":"evidence/20260720T190726Z-21133f9c/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:25190beb69f2f5945589638b29e43951d9e5afc96ec2658c33b49ccc7ca24e54","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:a385e6e36226f89f8bf38510fd7d6e665832ec84ae0e36888dd2574e8515bd0a","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:1c465c2a9ce494b5ed95bee022a07a75433a130f5bdf396063d3377d7d22274f","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:e8d533d85bcba30cb08d19bcd70d1d8fecd2be233b0b7b7da9737d4c39b3a103","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:987824749e4903cd6568de5081766cd37e279d1c0f2aac7d007ba4008242e680","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:8804da17ba7c6f6f5ca5c266fc8d6ce4570daf266305d400c984228efa23fc03","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:749021584d0861fc3c02d136eb59ae13909dbf83fb1a9a93248ed2a5d36e9267","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:7d7447eadfa69c34803e5fae60cd2aec381f19071353ca9b358b7c2c2e872e6f","evidence_path":"evidence/20260720T190726Z-21133f9c/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:683b73097314866f240e61f7175ab5917a5b0980a0efeae0782940463709d25a","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:847c1eeb20f8c39a321b9a0c34c3c55a142ee4ae486b1997843fd257c944c2bd","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:13de62abd3ed3ed60802769ed21011be01c168562389acc714c3702d6b2fe1f1","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:a019275cb1b204330db0e7f88914ea2aaed09deb3e3dcbea3f82d1ca03d50b8a","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:0269e508ea708fc30ae0698bbfa44fbe10f92fc33827827bd6b7af01053b0b44","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:523881f4e2693fb7e30632d21e4d302a2cd809d1c69ee09c1b17408484db2a27","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:7fd17411ec7cbaa2657a104450b61e59cae6c1f3506aa216a0296e36648d30e6","evidence_path":"evidence/20260720T190726Z-21133f9c/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":8,"timestamp":"2026-07-20T19:07:26Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q3_K_M","scheme":null},"revision":"bb5d59e06d9551d752d08b292a50eb208b07ab1f","sha256":null,"source_id":"Qwen/Qwen2.5-7B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-7B-Instruct","endpoint_id":"Qwen/Qwen2.5-7B-Instruct-GGUF:Q3_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-7b-instruct-q3_k_m.json","run_id":"20260720T184915Z-0943c9ed","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:cfa91637afbd5c21dd1bf16c0402182a8174a904061c55605a0cb41804133ff3","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:6c6d58780afa4106a9af7f31b3dc622adf78e81da059f256c28bdfb3a0a9b2cb","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:5024795e576400cb1574c17308ef56195ccf4ee00af04c44f46e8a7180476924","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:a7e88cc1421b55e2696c8ad543ba5e6e0820a0fd6b8fad6993ae8286a9254479","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:6e6a94560bb5db463431a3412c662daf5d49bf4278691f84c5060293f228231c","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:4eadd92a44958d9d684dbc06ff32bd2843d1de8be859fcfc56a5e45bc284ac28","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:079978be4113435be00a3ba6bba7ba1cb7b468a489dd0f51bf5fd422051d6680","evidence_path":"evidence/20260720T184915Z-0943c9ed/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:3ef4169be5a6dfea6bd8e651e8d836a91539ca8fd87f151bf347d9ac5ddd6a18","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:584524b7895eb23c85a5c492e1e6f8483cbf5f6bcf1f5235f39669f252baacf7","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:36bf124c2aba81505fb5da8238501b680038228926ac4373083e0222b1030cdf","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:57f6dea329a888dc14b73155f5ecfa963cae1f5ccb4aa2d59ed69847cf4f58b7","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:a45de391e4aafdf8a6a08366991b498d6047779e551b4d1a5e0ed72d06765aaa","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:08039d7b9548ea80753e5b80d5cd2df1cf035e23719fc7722163dff85bc9a60d","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:d03d9422f7d40391192c4b6e79ab8aff2564fb094cef052e95588b92cd525eb4","evidence_path":"evidence/20260720T184915Z-0943c9ed/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:db9278b788a65b0443a8b57a84c8eec43fdd35f3b318f14ae5c075570ca5f21e","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:b920c5a9065872c7f118c137b43ccc2ac3150f55a29e51c003f876eab5d98e4b","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:2fd188cb3addf55edf54833da4d38cd2fb5d332535520ffd6251416fb27ab749","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:92c76b6c1dd591744d4ef32b5e8dc767a1b135067942820f968439ba22ac4c29","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:964ec339c29805aeff403eed495dd1c8530fd6640b7471dfabaf5e4e58b1f854","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:82cdcd3f746d351e580f06003668213ef99756312546a46b60957409fa7fc912","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:d06d145fa9de048375775cf9cbe2748cb0855e1aba290667e5a6c872361593c4","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:94c838e90e4f9b90278abfe477b202e5d45d3e6321aebd57a2037ab604ae92b4","evidence_path":"evidence/20260720T184915Z-0943c9ed/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:714a5433c6e39260d57d30589572380a0ae5b1308dbcf6b6bc75b0b3e1a6a7d3","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:7e6f6e640749989bd3cfa8c9c361361074e8208ceca0fcd6af4ab8f3942f11a2","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:8f4a0f04c8650a97cb891bee13045c55d8f8d0a7bfea4d6f27d0bc39767367c6","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:50af043e157a69a9a586db28b08dbc1f59edc77f759fbe998c080d1e33e5a39d","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:823e4e830a70a1c3ec63a3e7e2a54aef3a397e3b174db518bcd66c212bfa600f","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:018924272735915f58d6a762e5744aa748efd5936e88e1151983bbb900fb7cf7","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:646ca0990c2b1e775b64fb9a707e68c27fc6c0e6417c63a89caaf58d24e7a5e3","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:bab276550fcb1fdb81a6aabb5ea5685fc0cfa9c6d869bda05ed6d13b4a5ea293","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:b8d650281409371852293ca84a53f776ecbcb104680eee4a7cc2e21351d51deb","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:2261a66177219bade69f5181d120019f65c36d78cae50f8182a9a47aa078dade","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:717853707e9e7ccfdaa349ec2884db6e6731f8721d7fe82a12e7c871f1793205","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:cb531731ea2eb4b3668f291d51d7d9e47c084239d0cdd9ec31e6c04cc057bc09","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:2368f9e21a190eebabe1d23b3cd3c4e4e172dbe8eaef98095464ba884418e1dd","evidence_path":"evidence/20260720T184915Z-0943c9ed/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:130aeaa740b17cd45bd451aaa0c417c5d416f5d7a1ab957d9fc180bcec2f1b78","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:edf6ef5239c77cc05848787ced4d634fb2bf06154445d02a6936620bcb804640","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:eeb1ec6e26bbfb75e0eea485642c7bfe1f19bab1b1b7e02f9d7a62c6a26d8d8b","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:797f13968a1d067e15e336c0f06298d336d27b5a5eba45f6a10a4fc0ba08665e","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:50cdc337342eab3ce81c93882ce12b2737fb2814874a244bcd05fdcc982acf46","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:b36a3f13aabb821364d8a01070e7a4d1ae312796f1cfb7fc19da71f7d184c774","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:d003f011f566969fbdd5410b69744599c57338a8442584b84495e460350bf6a4","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:bdc397167f289ba84274f94d09c2d67874c88c45229d8622609f55188e5fb486","evidence_path":"evidence/20260720T184915Z-0943c9ed/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:c5cb0d15b3726e80dd34e39e5547d09a5faf81746ee4581ba63e88063a70a2fa","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:a040d854c9b420fbf241c991861b5933108deefc84752b267b2f18546bed33c9","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:a2849370b32f9fe15023c7e3d619d3cdcb582b2ec47ce2083c4c1047d90a95c7","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:016e57c3698a38d1a7c960a8ac77c80b46567b930a7652452867dd7c92a239d0","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:0a4f583e04ced3902d53a38157df3884e0bf73d06ad4cd3da889427dfaff67f6","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:1f8077caa8dc406bc061640409d0e4c6e17285a2d0a4bc60616c8682bdd8c9ac","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:a1ffee50581c433b9d4caa50a6ace2296a14ddf3097902b6bca3fc713997771f","evidence_path":"evidence/20260720T184915Z-0943c9ed/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":9,"timestamp":"2026-07-20T18:49:15Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"bb5d59e06d9551d752d08b292a50eb208b07ab1f","sha256":null,"source_id":"Qwen/Qwen2.5-7B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-7B-Instruct","endpoint_id":"Qwen/Qwen2.5-7B-Instruct-GGUF:Q4_K_M","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-7b-instruct-q4_k_m.json","run_id":"20260720T184338Z-e39de30e","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:b179d0698b819e4c0e42480626e4c32cb528f61a7f9a6cd50e660ee13d0f1d6b","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:07c3fee269eb72c072955838b4f3d00b53e91abfa79fd8fe270e832747ad0537","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:76be3b23c24696d1063750a64ab34ec2b58436db1052ecf68e50b98bcacb2863","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_order'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:6be3ceeae74d0ff24079b7fbe59e557bd5f3d0f9675abc7b467a7e508e41d5b8","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:bb25743bd9ad18c22bd7a5869606833f9fe835282cfa7789168a649d238513d9","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:6499b24eaf58bc9a271c6290e8fa4c03c2d443bda7560c9d5a2e84ebf5cd6cfb","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:2f5790e2a6ef733809ed65a1538c539b589f372c19bff4dba3d2e4c4a9d76db9","evidence_path":"evidence/20260720T184338Z-e39de30e/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:8893be7698bb6d6db5b7b8d125c398bacae9efef22859bcab9cfdf7fb981b95a","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:52247d2f25e8f780656b0b2ec69a4835fd51299609dec66e4dcb8407d6b01ef9","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:bdb61c17812a305873066b4acc9c3b507b23f4e26397dbe92a08b9c6c7528c20","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:269efec2e4f5dc526725217c675764052ef0d1df796698b960624d4422396bf8","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:2fd1002dbff00e8a744e16ff8039ce01927c0f9e270021349f31e45a87362de9","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:41ebd959821cf854d88aa1245460d4565e15f2df26063bfc17aa386d78cdaddd","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:cdc513c7c5375e60b3a2dee71953a8810d384cddd1f5da07566691d7a5cb99b2","evidence_path":"evidence/20260720T184338Z-e39de30e/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:6b662f374eb269b6a09e909d1e6fcbf80065f0066e0849df9012dd3a2f9bef7b","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:d3d0f2e2050390d5c0bf56c9ea60dab7f71f7e9cc980f8418c4487e9670d7f2c","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:c94952e87c0d8430c3f8e2e068ea363b6e497884820a38ba533641c3119497e2","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:e134393b72294264e034910f9d36de1499bbf19126c9fd3913058ce53cf1e5c6","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:dc19352f2f3537b97ef1373e9a6afdf697fbc3298f11c42006a2ecd7b725c544","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:56273ea0786256d4e4487541420f10d189fcfa476042c9ce674af749a3f689a8","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:6c221bac8606c592d731bb9d110f5e8a380b0b8a67b442d206f137b8bdb3066c","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:9aa2411863aab9f46755653fe334a247915874bac3da4ebd587ab764f8177d4b","evidence_path":"evidence/20260720T184338Z-e39de30e/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:1502994ff088fe92ba10d5cacf5254258d7463fa9f348c44396142f15d895e54","evidence_path":"evidence/20260720T184338Z-e39de30e/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:c567ca9c2e13d1d54d6ed7a1bc3acce5eb09557f467c8516f0f99bdd9ff59be4","evidence_path":"evidence/20260720T184338Z-e39de30e/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:82347b0b19b8c33923f5025c1565dcd43531b566750dd388033123deedbbf507","evidence_path":"evidence/20260720T184338Z-e39de30e/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:2c29ef6117b655c4101ebaf53a01df251dd8f3be7675fb90d3072d3fb3ec7ebf","evidence_path":"evidence/20260720T184338Z-e39de30e/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:343f10ec60bee9be00aa5cfc83867fd7c94994838df7cb3042c506bddfe7a1bb","evidence_path":"evidence/20260720T184338Z-e39de30e/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:f226690aa71bcad7f57ce7091dd2bcc8cb2259e7f93a2582091f19451a29c3f2","evidence_path":"evidence/20260720T184338Z-e39de30e/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:4c7c771489134177799ffbfb1dc4e28468257286b26d45e54fce5ba01c420be7","evidence_path":"evidence/20260720T184338Z-e39de30e/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:e250e6dd39febcaeb2bda5d92600ece10cda9c9d0f2c67d3eb59d9a974ac69aa","evidence_path":"evidence/20260720T184338Z-e39de30e/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:78aaefc69f4b19e76e5c1f656b794cf900c16ac934a4770f386de5debdd64ee2","evidence_path":"evidence/20260720T184338Z-e39de30e/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:2c6b1301684b64220d350b1ad48ea5c773cb4f0fc58df6f5c6c6d614c1c12097","evidence_path":"evidence/20260720T184338Z-e39de30e/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:2a2be4ac3db8316c8360c2db49dbdbd4a638f50c94f155666c8162e9d96c4fb5","evidence_path":"evidence/20260720T184338Z-e39de30e/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:82043665a674de20bf104d528423c37c58169a788fcdc8fa7a7ca3082b68ca5a","evidence_path":"evidence/20260720T184338Z-e39de30e/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:28204b533a59f0fab287259b1b9b2c0a1b7d4be17a288bb9e89a413ce10ea021","evidence_path":"evidence/20260720T184338Z-e39de30e/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:ab17f2a4fac870717102fd0478ee99bbbb90914f4159174055aaa628473c5d53","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:e1a0b2cbea2564af8f60aae68a6cba0b00a69dfdc7481daaea17d8a7301baeb1","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:8a0b8f6a78e951e61bb822f15f36e49f14b129777cdf9a4afad640dc41e26901","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:4bd0d1d186ad10d94536820a1a82d35cdd7cae557cd73894a9792010679087a9","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:c8199e9ed497e1edf3837053130fc53390932b6e055628ddb7142e8d0b1156b0","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:09b681ff95ae780aedcc61751670bd58e5258cfcd792c87dbd42a735181d3d80","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:d520dc929978a5848f7c8699d2143d8d6e9ba01d8112248f40b37c640b077dfe","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:4611f216a228507690b7e313980ad4171c3c2d280e7377665f9da896d5ba737f","evidence_path":"evidence/20260720T184338Z-e39de30e/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:0cc4bf70f7e3ab7cc7e5de8aed45f74a0d2df184168957dd50055c2402bef02b","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:5d459bd1748e1e5985191bc66ab3742b717dd8d40e8c0f51e1770c49608f414d","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:27afa72c1b7c0666515e73e23bf1a6fc1987531687963dac1aa3704c024d58b3","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:2cf4b7fb34dee1b9fc071bfe0cb06878263c2430157a58db912bfe921f54c89a","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:b6df91299eb66f0f008fc5723063da7df88a22bb9998b5d46a889c703e6ac32e","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:1314d28eec13b660b717c8178ccf4a1f1b556463506085a83f97b597076c039d","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:0336910bfe6e56fd25b5a6d3600943bdf97040b6d12062e383d4977559e86206","evidence_path":"evidence/20260720T184338Z-e39de30e/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":10,"timestamp":"2026-07-20T18:43:38Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q8_0","scheme":null},"revision":"bb5d59e06d9551d752d08b292a50eb208b07ab1f","sha256":null,"source_id":"Qwen/Qwen2.5-7B-Instruct-GGUF","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-7B-Instruct","endpoint_id":"Qwen/Qwen2.5-7B-Instruct-GGUF:Q8_0","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-qwen2.5-7b-instruct-q8_0.json","run_id":"20260720T182604Z-d59df055","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:e646a6d23674353186d25178320c011e6ccba65a6f957930012fa148d494c1a4","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:de507b3e4bf2efa8d7407f0e144a70c574685207be8b1a98f4dfd2aabb1864aa","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:49f43025c1e8dde42601a3215d1527d8f83a850ba45f21e4a6860c6bacc2c5e6","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:7c8fd03c0b767f27a824c8dbdb233c72737bc48fc852519b235ea2cf042b43dd","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:29db47dc337388488021e4c8fe89793b52db4f7adc513c5d251b4c72b6159778","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:32a581afcd10a8b3021464ebe070895c876f2134b6fa55c1bd3048d1f89e29d1","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:1f1f77a4f7b64429186458fc49d721687f27167176c98f2d4e3d41bcf2932327","evidence_path":"evidence/20260720T182604Z-d59df055/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:380864db4f8346be683e28b53f16d475c02eef9f850862e6b337f470a49765e2","evidence_path":"evidence/20260720T182604Z-d59df055/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:adfae89dd957af344df21d0a897f9bb5fb9ba2e410df5f1ca0d7d205574327fa","evidence_path":"evidence/20260720T182604Z-d59df055/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:1d771406d4432f1e4fc2e1e6c9702f20c6f15dfb7def6d54b491b5b54cd18dc1","evidence_path":"evidence/20260720T182604Z-d59df055/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:255ef0e3a9f5a8697e4a3388803f847851bdd8e1fc8ad507932d5abbbb9da7d2","evidence_path":"evidence/20260720T182604Z-d59df055/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:3f7fa183cf71d2f95e2145d44369be447726b941aa2d7c3fab024d9995e2e2e7","evidence_path":"evidence/20260720T182604Z-d59df055/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:705536c02f9e37f074eab156067923610f7a027f094a0e4c021b090e9788fb25","evidence_path":"evidence/20260720T182604Z-d59df055/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:f273d92e834125f180863d43a5db5d2ffa77c93301c9577cd9eb8e43cb0fae8d","evidence_path":"evidence/20260720T182604Z-d59df055/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:dcad80c6fb0bf0dcf42dc6a92b66e45c6b7d723e5ddf13e610ad3d98e370745d","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:2eb49ddd34c9048e280ff4b1c92fb775a89df6792e477458cf3b23cb8b66e9d1","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:d6fb3caf229360ff4337a173e979ea9878f295e154ed51772286ec0611dc2757","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:cdba6a972b877233a11101a2291d14ac32ae2d8c1bbaa9f19f79ec55ac6af59c","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:04e228e6a58937084ca556f69d42970169a0c38a9df3b098a0afaecd02c9b3f8","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:ff383a34693d1207aa15372ca3cf19bf692755b80f5557d95a35f98be2d95967","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:b6b257b6229ef5fe5881d2cde00de593cc925df205eb774d47e3fecea1ec0a3c","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:8f65c4040842318b3ddac1ede2d2d6127c9c90783dbeac546100c58b8dd2c6aa","evidence_path":"evidence/20260720T182604Z-d59df055/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:3c45b8b725cc11544ee589841fad0ef20b7dd299ecd4b54a199fae95f6ab767b","evidence_path":"evidence/20260720T182604Z-d59df055/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:142467b4e1b8a608921cab6c88d729cbedc8d544d30d3fff872acc1479dd1518","evidence_path":"evidence/20260720T182604Z-d59df055/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:5d98512ce1fa72264892b8023de9d6f93d6e80f1c355bbb2a23d54957aeb0ef2","evidence_path":"evidence/20260720T182604Z-d59df055/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:fd50ba68e9725e074814543c9838ca62825b7eba3e7749fefd91460305228a16","evidence_path":"evidence/20260720T182604Z-d59df055/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:e52940c587eda978892b4835c856640a53c1f12d52bf4e759f622697c40fd3b2","evidence_path":"evidence/20260720T182604Z-d59df055/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:2b1c500ec907746d23ffb101273c06a0b7ce39fd675b3fdef5c5e98366fd348a","evidence_path":"evidence/20260720T182604Z-d59df055/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:51868513be7bd4f223bb5ecb068149ac47b74e4fe23f0d36a9e68a814d2f5f30","evidence_path":"evidence/20260720T182604Z-d59df055/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:b9c3a745ced1415dc1bb6a1ffdad7edcb89c9d4efb6555bc5c0ce37075053081","evidence_path":"evidence/20260720T182604Z-d59df055/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:dae4b086290b6aa24f77500160dec9df28b70b0940428b8397aaf006c6d44556","evidence_path":"evidence/20260720T182604Z-d59df055/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:044c989730577cd58c8f587e51c650cc34fe3f3cb51e39e3bd367fa16ecaada1","evidence_path":"evidence/20260720T182604Z-d59df055/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:b1761405b7ccc6dbeb6445e3202f3d555c856b540a679e5bc6a01a48577e66bf","evidence_path":"evidence/20260720T182604Z-d59df055/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:3d54e689beafb2818a4c90c56f805203041c752ea72f6ce527060d14dffa4fd6","evidence_path":"evidence/20260720T182604Z-d59df055/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:378583ac40e2cded42097a670cb9e121eed8d92d3b7f21dc1595a82a943ac10d","evidence_path":"evidence/20260720T182604Z-d59df055/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:3bc9f0fd1bf30274981753c109c89f7c81f257f5783d353dbb1edd8b0ff9e867","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:e9e0ec4adf9e01242b9f89204d036f5f15ba6dc840ee465519d62979b02db62d","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:9fbc0a705d181be9c3ee7e7175242c4400aef4ec7fea82c0b40c73b4686c437b","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:edc027a76ac6821bc544d242cbec9c4497d4f44992c9b9906858be7b83609050","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:d200b4269959fb9e15b3213c3d22e2f3894894869eb128055f71211d57a2c14c","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:01df3bbc8a7b00ede3f943e5fd1132328769f5418bf07ef9671f6f0b0182923c","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:9b2b0607daf31c0d51e60430e3111894f0a2069ccc94dfe87143c775429dc39d","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:37d974a590d2c02cc1cdce859ac34839df6180b96f8c6c1c3d3db790fe6d7e0b","evidence_path":"evidence/20260720T182604Z-d59df055/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:50ca10ec872722205d0ce32ef32c6617270d889b42a19ac32f21b733e58f7818","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:829622d348ae401437b69b37f8128832fdb92f37c60eb7e7175c7a567985f57a","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:275dfdbce3d7ba9b137b36f92a96f599a01c300d2a2f102d8055c5b19a9c4572","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:3a1929cf43e17093b6000d8649935ba4e23258b9f145ba96f1a0aaee50492c5f","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:ea1c557a6a201c9bd04de35035b7370e6537b5e24b011799c172545b42f51aba","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:a58b2e58580ffda9672ca9e9ab7ebbcdbc2011843aae267dc0cce5a34e374482","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:18a5c7eddcc361957bc262a58f105e75b9702d62a75b72a848503196ca2b1881","evidence_path":"evidence/20260720T182604Z-d59df055/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":11,"timestamp":"2026-07-20T18:26:04Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"gguf","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"0e8a7402ad3f90bd2d65c8ff04f6b6cb7ee3df7a","sha256":"sha256:33706b165cd6777e29fdcd777ba1e09bd3b2006b428014e1bad17df3902ec1e7","source_id":"watt-tool-8B.Q4_K_M.gguf","source_kind":"local_file"},"canonical_id":"watt-ai/watt-tool-8B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"watt-ai/watt-tool-8B","endpoint_id":"watt-tool-8B.Q4_K_M.gguf","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"verified","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"llamacpp-watt-tool-8b-q4_k_m.json","run_id":"20260720T191115Z-c7b8ebe4","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:f240d51ee48ef283f257f709e43b0ffca7f009f972f67aa3f82e784429992135","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:44a78d0ab33f82b3693699a1fb453a727b339787510c56152aa27745683d1c09","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:588535a55a1dfb71008c1fda8c297575c95cf90aa66f9d3fa2d58000e3b82265","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:7af9cdc1b36689c6ecc34521160b84fccb178380aea0c7de21816608ade32547","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:7247f8873bf96a0899e686b08cf2890e526cb3bc33099f4b23747606229e55a5","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:950c743d875fe91cc471c5650e7240c25c021b1ce9399059a1019952c10bb9e7","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:eac8bdb53716bdddbf11766c42929f26725eb4eba690d765cedd3157842041d5","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:3234df0f4e9487229ad74aa0cdcf4876dcc66880a9d62b063990393e808bc0ce","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:82d25bcc8413b0fd9a7a2a20fcc5d353c086fa26a76050d34b37fea3326b2566","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:0a98ee5cbb314e88bff4189686a49246c085d5507db5545b7d40c9892c4466ff","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:aa5544d687c4d3cf6670a81041260deaf897248dbe03bc38648f4e0875a4769b","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:d71ac4ee05f7680d6ae77e8ec21633f6ca863607b8fb132a41fe92f754a52684","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:3be778adf39007044ab22d6176e5a7ed2ba95871aa2f9236fef0b7fd21843bee","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:ee43e5c44d8844dc03d8c3a122a7499dfee9218b7cf5589516b60478f171a50f","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:bc921ec95b775998ab3648157f90ad885f2d6cfb5399f855a3c9dfeebeeba05a","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:25095457488e70114a67d8f166eeae9988e8e13db865f611968bc433d0217a9a","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:64041b950d6cb6a1da5a7296dd04ae0f3d0dfd9ffa9baf244121580d2608a93b","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:6d3ef030d87232f60ac312f6938155cbd3805fd45ac5df0ec6f09c196f0f6e68","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:aaaad9b07621f5300a60a78e214cf1a5fc66c46e2ba604712c836d248c542aba","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:2aa51212001a74ecfbe49adf920a6e7b363328cfdda266cf616c06cd8ebf8b29","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:2a75480fdb9778e3d7f5787961ae94c7c7818ce13344b3d799b2c6cf8ce6ebfc","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:90ca7c7f41c03c3f6a30d382d5c6e69e01d5e26aae69c88f5f9fcbec96ad1d48","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:b986db6f2ff6cb58d32dc5c4646968c8a03a677b4735e857e408175ea7fb792c","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:4a8a488e1ace76c0d1a86e57198251e848d38930d30189ce6402cc4a5eb32cf1","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:1a9d9cb28a738fa2ab49e4a88a75c414b255096eabde74977125cb49e125ac2a","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:a861b842294f6e5d33729ffaaa8a208228a3f8c0c4478972e9327131df49b314","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:959d26b3c67c00c118002ef21195dbb5768f1695f8c3c2c7fd5ee6f15ab14987","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:967148a8f07d5fc76adeef7f63e97dca72e6831b4d1071fe2183628f0199b268","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:5a553b613f675a762abf0dc9c68a87440eda20291e5aef496f887c7b6c495bd4","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:8b529f288dd8d0f3734a65f2f7baa9a9023f2bc5b0a2da28a04037787c1564b4","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:e6fb7c65b317a8f18a07f913542624b14cfafdb7428686e9de5fdd69e55cdbea","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:913f463e08c64ff8a245eae8410db10b59d692510731ab04fc92975768ad1068","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:0a8cb729b8ff6e73f3360719f6f54dedf972f99ae423245efebfdf7810770e25","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:142a251771f4abc92b73f068f7c3e4eb88e87529269a532bb983ed224c43e4ce","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:4d54a344565cb2557cce1e5aff6e141445c86127b6151116e43cdd44437fb22d","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:0b9754f694b8efdd20c347c2648a2ab64023fcedb4b2b56332feb3e5d1275220","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:8cbc88bd4499fafa4cf1faaf6670309071da61fa8e287f8f62cca2a5b5fcfe06","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:b025e51d5d38a8b88c7f7884cef066b3d5a4283a66d609633fff8d6627c79c32","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:6acdfb6b61e1d7b0a94b313ca60e0256dbd6956f231f1de1f8883b38677a7ec8","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:9df22302b6075109f1a6ef75af0a64eab4848f26468c40a08d454cd1ae89d89b","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:7fabee8688867475737ef0528668325b2d61d63b2d31014cc0ac178f21ef025b","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:27aa9086c35d54da7575c4d8a97cd56b584c4a6febf3914a2e998f253bb3f4cb","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:d587e0c0e578fc2a982728e2f2241d8965c0da4374754d764dbb19ef8f0879c0","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:389f74091d1cc42887aa13c35151769845d2125f7bdc2ed4af13adab400bcb39","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:7f7bd0a748c22a8be55833b8799d30276be97e150d59df06d2cdb23edd59f6f2","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:771617adc40dafec6daf18d3f685b599d6d04b4526398bf927ccdabd29f62eed","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:6ffc06a4932ee562588c156aa66d6e8a46c8e0891d648ec4c3a1ed0145211820","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:97d48a537723309304cd35e269ec0e15b496e4ea5818ccf4c1b69c3c7f5d5cbc","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:3b9302de3a860fdde3892dc4734554b2c7e7319c688862ea8670f3ea481aa8a1","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:705b0f7a3c549ff2cc969204b5738ecc6db85ae4a2d790d2062e5e4f37239cdb","evidence_path":"evidence/20260720T191115Z-c7b8ebe4/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"grammar_constrained","launch_config_sha256":null,"preset_name":"llamacpp","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"b10050-b15ca938a"},"stack_index":12,"timestamp":"2026-07-20T19:11:15Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"4bit","scheme":null},"revision":"241a666dad6cb93c8ff213d39a7f34a36bf26db4","sha256":null,"source_id":"mlx-community/Meta-Llama-3.1-8B-Instruct-4bit","source_kind":"huggingface"},"canonical_id":"meta-llama/Meta-Llama-3.1-8B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"meta-llama/Meta-Llama-3.1-8B-Instruct","endpoint_id":"mlx-community/Meta-Llama-3.1-8B-Instruct-4bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-meta-llama-3.1-8b-instruct-4bit.json","run_id":"20260721T160256Z-93643fc9","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:ac2cbe801215293d3b2b70f0aa8c590d82ae8880fc886c24740c858c60894811","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:49bcfe25d46cb864680fd761e2254a3e441c9fbb142c75ad45ebd85e5b75503f","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:2e6934967ccfac8a59d1fee3666c37cdfe6c47aeb4996af45e2eaf7d5a255e7b","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:acef808a4ebcbb505fda661f429978dd15ee1e78082d2cc0634a11a558af539b","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:43b5931ecab57cf607c74e33ee821c8485d95a0042ba521a7e08eee2fc466071","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:4ddf2f54596fc43d2b0a375ead90b58b56d7ad5535371967eeb1d0726a888b9c","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:8ba66a8f4845e205dd5280888a255d4da891d089312239cccf104749ec05fafe","evidence_path":"evidence/20260721T160256Z-93643fc9/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:6b3f8ffe8fffd164ae2dbefd2c347a881bb3799560c1cd8e0255fbdc1f359353","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:533f226ccfac8c5fec0fb6be8076f003f3bad890798eaf35293e68c47b7834cd","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:7ce66605e0385b77f053b518e88df0edf779dd73d58867129680cc27aeb76bf8","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:988c97992b488435315fa3f4c06207244feb1c1ec8a57c6ca9c49ad26bf4219e","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:bf729fdbee9e770e2683c3498e0e6117dbc96fdccc850daba2cbfd0234f06269","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:c9ee3dae964abf723ad8ea49cbffe7b89d6517f8dc69bafc5e45c02f0893f04e","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:f94dfad9c732b254507a870a2e010fb3ac8efa25531eaf1c665e3b5c4e389c9b","evidence_path":"evidence/20260721T160256Z-93643fc9/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:1a323d386ce60bf474a5714d66cd84f5ebc9fb5f3b9b27d4fa22b251411ad441","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:3c22274b0a3c7a6a3674fd4970b24fede8f2366a3ec143bad575eecc3ef3b472","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:57a00b8088c3681b69b719ce17c4135acb044946ef540f47c07b0420192195c7","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:7171ba270733ae8ffdda19ac65178a824f974e7c9c5c2397964536e5bae1b30a","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:b82430f75d0140971ccac731f4b7a594bafc80a893843e40832934a438d6b756","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:c965cd366b0b1a01bbf2427d292f471f538a10f31e4cb5b70cd6b71af8092c54","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:16e819aac8a19dc2fd41e81edb2d8a703033c369b4a5cd61f4d9b3ef6cda09bf","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:4b590fb38f33c13254f4323e4d56ffd551b4fb64047dd3e55027fb4d18c63d7c","evidence_path":"evidence/20260721T160256Z-93643fc9/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:46e5dad607f8fba6a0b330c049bc073951b0b34e52d0ac003716e96a952598f6","evidence_path":"evidence/20260721T160256Z-93643fc9/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:c66733c784162ab59cb3fe56c0a173c2367d2e26266fdc14edbc53f0eaf82d46","evidence_path":"evidence/20260721T160256Z-93643fc9/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:6b65a8e3f9866096469628f5b118c9d6fe0c73502d518a534e86bf1b08f7f7a0","evidence_path":"evidence/20260721T160256Z-93643fc9/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:537ade7fef72a9e9c5cafee9599a20065ebf207d545e6b767eac7066ee987878","evidence_path":"evidence/20260721T160256Z-93643fc9/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:b8fa426731b5d333d40d682eb825f6c09f086f97ca5cea7c02675da0f8be7d49","evidence_path":"evidence/20260721T160256Z-93643fc9/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:e2e02264c6514f0a593837bb0617bc6d8749e82843807950540f1e34a196a08a","evidence_path":"evidence/20260721T160256Z-93643fc9/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:9cb0987c2f6df8b7273cddd6743f9413063f204326fe1fca0c0787875468c442","evidence_path":"evidence/20260721T160256Z-93643fc9/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:d7d4ed2bc300f29828775d96623d90bf6815e3c97bbb5f6ab490cbf09b3b395d","evidence_path":"evidence/20260721T160256Z-93643fc9/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:78a67bddab5a31041ef030c75271b9ca0870bb707b217522e41eadb214f1b723","evidence_path":"evidence/20260721T160256Z-93643fc9/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:7fd11dc0938c061fe95e5607c665e8db0e0d6c7d0d697ee8c36312801c3a01aa","evidence_path":"evidence/20260721T160256Z-93643fc9/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:509db171d0cb6a55cd470cc0e4cbe39c5faef49421016e8b3c5a36efc88f79f0","evidence_path":"evidence/20260721T160256Z-93643fc9/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:f5bd57c120c208a930b26c170c9001f0a712a46c92f9867bccf2d990b7451697","evidence_path":"evidence/20260721T160256Z-93643fc9/single-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:9ace7af5ef129efe383e78f3fcf592b6c0d73c19d2c8891332b6a1ae312bf1a1","evidence_path":"evidence/20260721T160256Z-93643fc9/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:86daf24afbfd25e5483c0025576de47797ced0ce2ccc1a543a2d25ecfa25e37d","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:5969df8a647cf3cb11916b986204310ce42c611991b968596e0617ba24daa647","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:a07e4d82e9a2d3fdd895e3c00930aea78f779369dd44bdf54535e73439dc39ee","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:276b0d98e316818ce5e168dcfa547b6907611555966ac487889fc6ec2e38c3ed","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:e0081cbb8bd4bb0694bf2852eec60ccd7df0974229046d49485be160e4e68e6d","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:9f211406ac1a43af4618dce91da920ad32c9922485834d4c7aabca3ca1ee5505","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:5499be24861fa080bc92757fb67456d6776fb4018d999b89a9e5c6e115746730","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:4b57a4c10ed32f3829e23d30ccfbdd56e8406dd7bf44ad7e71ead9308b67ce08","evidence_path":"evidence/20260721T160256Z-93643fc9/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:4b38df372c60931dceaa5cdc46f215adedf9ca9baebd0a9195fd4da613b20c79","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:9dde8143a544c212b5f6fd07faeab560bef3b8058060a09cc9e44401514964af","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:6029b878ad4749a427d16364692787c9a11d7b43e43a88092446ab2e3a956ac6","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:4e7610531eae47a9b5fb7d57409cfdd5688cf5b38c653c4ea7dd0fe95f566e09","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:930c61e5958e1529c871e9f0736f6bf643d4442009a3e71082bd5195502e79ba","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:8b2422b995ddce3f853fd323796fd5085d595839f48d047d9800927a5589c9f1","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:aefa21a38ceb500cd4a571e72fb92a82783b7a28b46f9226c6689f613219b0f2","evidence_path":"evidence/20260721T160256Z-93643fc9/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":13,"timestamp":"2026-07-21T16:02:56Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"4bit","scheme":null},"revision":"a4b8f870474b0eb527f466a03fbc187830d271f5","sha256":null,"source_id":"mlx-community/Mistral-7B-Instruct-v0.3-4bit","source_kind":"huggingface"},"canonical_id":"mistralai/Mistral-7B-Instruct-v0.3","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"mistralai/Mistral-7B-Instruct-v0.3","endpoint_id":"mlx-community/Mistral-7B-Instruct-v0.3-4bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-mistral-7b-instruct-v0.3-4bit.json","run_id":"20260721T160520Z-e6bf3c09","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:2736de70399012653aeac47a34966b7475e96fe2d7d118d6a1d26e63b2c51450","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:4ad90400358e9098f168ddd156ae878e00c83441dcc69207dd0a0153b066ed48","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:f60dce0a1e053ef1cbc8f85354c3236513f678970dedbb99f7c16868e46cf240","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:f017b7d5139dcfa2a39fd2f1121de84b61b479777db5ddb5efbf57f76ef4d7b0","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:48bad57ada7c8113483ea846cdaf98285f18365e8688dbaeb80dc2a036e8dc80","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:385a8fe027ee552ee841ef678d37a3722d26e2f754ea9609bffc9ce1a22ab6ed","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:5c66d94c5971a1fc1c38d74439fa87e5c32881c132b9849954821ae69f149608","evidence_path":"evidence/20260721T160520Z-e6bf3c09/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:a8da506e473966c7511e7c80733f086047f962072c01b19586a72e0a8d1d0fe7","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:a32b52f280b4f2fe53bcc297d1efbebdf2288043e1d260a6e940a66f026155b7","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:9074d4963077f10eb57a10e461568caa00b09bcd9689a0e3acf32baad31afb2b","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:37b068e744030d6596d80933d50a59a081f4a81bc7fc812a06fe1f41cd5dbe0d","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:e808cdc82d97e6b9bdf9dded14bf4e9c8ad3809373d720442d08208d784b4e14","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:3335633c477976b19963c36e9a3dcd6ceeec61ae75c4403161c648fa147802de","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:df7b3e719be27a0c237dd798c17ce83761556e372236c8379f71b36b4aa52d92","evidence_path":"evidence/20260721T160520Z-e6bf3c09/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:3f3989e0f85806b9f5fffe5f80982fb00a83469d0b09fb1f875eafb94d28aef9","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:b71d695ab2c3569deac247c543c0dd73f8ab388e08df897e3b1818a863e8b6f5","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:d3d89f72974c120ac58730de0a34c5c74a7552b39441485b4e5a4ce9d158d30e","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:2e6acceb310bb642ededa0956d497ed368f03a3b6fe4c0f6613b342743159b31","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:6181a9f2439be8d5fbf4997dd25a98ce88f1595100928be4e08648217855e8c6","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:cb76b2af30ee1366a0d1a0ecd155065023b271b0359f7723c9b7f7d8879c94cf","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:3e0c2be693d82fd5842b73c265c019981b18d443c970cd698f45c85684625182","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:9304bc2982b88eec58b4ac15be0a755394da9f212d671bffb515cbcb16df14eb","evidence_path":"evidence/20260721T160520Z-e6bf3c09/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:7987b521786b8d433d49f198404c2f9b3afd183764b0e68a9a0e36a177a79b30","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:487255741fa46f3381f1a1363e88a5a79bbe670a3de398fbb8618cdfbf690ba2","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:71e8feb0642d67aed7ad78fc8dc9f32912103e1c5a08e44b49a9b49a3cff7afd","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:c6e83d17f1f9c51a4a15b99af7e1af29f1ac0b35e46d0ef41e120b8ba4dce23d","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:8c8758f491469837c1e3579aee575478eeeef7d941df73d9a5417a9b878661aa","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:a97cffd02358084785ad2370e7560287076397d3f34ff6b8c751cb6e0ee752a6","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:5d18d6ab45f19a6ed41d6c6e1e9154e32305a6e2a6f790eada0864493ebfa1f4","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:b44f670d0b2b1a542cbf45df66d16172a6e03cb1db4547676b14393288693287","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:1d27b5203acec122eb82c4b299634faf6f3bb96d3777f6f79c32fcb541dd2575","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:2ab4fbcf30e84f8f60c47839049a555da13724e565090ed82cac94e043a517ed","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:f684fd1fb235ce25a8e84fc7c7054e15b5f34787d1fc1ee1aaf5afafd67d2d70","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:3d5aeebe9dda733414421ddb63ad5f8e3f70d272f1e111a784fac521db4bc128","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:946142e1f3ca9ee6e07388e032698ff8882baeffee3a2456af9547a96a802f0e","evidence_path":"evidence/20260721T160520Z-e6bf3c09/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:ae889b58c3c5b0b59333fa19d7b2bb563e10a8f4bc259a75e0db42616861001f","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:cd409aa71e4f02b820a63c7523f39816e6599da6f3a0e151072b611f740bd981","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:0baf2fe10eed20519b1649b2865ea341f3d54c2bf9dfaf5381525eaf25a71ee0","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:a016e49d5cf2addf0dc99864d7621323738c77b6088b8e59e27899567d76ca8c","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:3c7573a040667bbcbd0c23a0dfb6eaa37df397a9716392e7346ee113297934c2","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:b1d7aaae8272a27f69530153b2e67f8e762ca88c23f4ce6617e890ff8f7690de","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:7efd702f20285fc750aaf5727f503345e4f69381696b98255d24d995a9a7f76a","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:2d56afff2e8658f5ffad35a9d16c1aa1f8b2880832634f4995495f40a91acd5d","evidence_path":"evidence/20260721T160520Z-e6bf3c09/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:97447080b169f926b349be70e0a9f62deafb31b4b8412466eef2f12049f3b735","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:0a16930f8a077dd668733841c69f36b6db4dc4813821c96c8633dceafa839850","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:9f02ae3a16dcf8d37946526e35c1d1e9c774ce772db0816f5044c196ebabcae5","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:23e862b0ac63d4c011bbedc4a434a5e77d6a1bdbc9e9503d8af24e09b6369a36","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:b0dea822151bdca75b56db7ad2a5c7a89d730eafaa593640ec916d77c5aec903","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:e8709ac5d22fa9d41e42367daacf83eda84ba078501aa901e60fb70a6560fa7e","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:eb54bc0ddf19e089627890cf6429f79e48b30f7806e991829bb1a06ac0d1e1fd","evidence_path":"evidence/20260721T160520Z-e6bf3c09/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":14,"timestamp":"2026-07-21T16:05:20Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"4bit","scheme":null},"revision":"ac1c269cb4222a4e136a3d09edad301056c1f36a","sha256":null,"source_id":"mlx-community/Phi-4-mini-instruct-4bit","source_kind":"huggingface"},"canonical_id":"microsoft/Phi-4-mini-instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"microsoft/Phi-4-mini-instruct","endpoint_id":"mlx-community/Phi-4-mini-instruct-4bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-phi4-mini-4bit.json","run_id":"20260721T160343Z-771ba56a","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:d6082e43b028ee03fadcddd040d47cd5b0ee4b447ee11df8b5a550fbea8eb003","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:e471682e8ab20b7170ded230f63033d5468e271cd4ce09fae7f22cd4c1be19e1","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:8793510e8d42df329cb9af5d2f1619a41a96e747728df739822d345cc2f8453a","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:6d359d03756014c828e4ae1833aa4bdbc8161b97a9e3fd10b2aafd98a66f8d5f","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:e9b2a465415e67d59405805fd8f9e38d6cf386bc30642827a5a5e34ee1d46f43","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:c383cac255ffcce6422202ecea34af40650b6f2a00bd217948472b51946ca919","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:6fa61c5d0858e5ddbf17d17bde6c7dd3dbfea0f0bb8671add486591a6c7fa3b8","evidence_path":"evidence/20260721T160343Z-771ba56a/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:8e08dd244517dd259a85e4e43ce9f6abee9f494f3f036cac14a16f9703283b5e","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:18318ff53a52fe87af05331bed567e1437c2f48cd9e7e63d39ec85e75c595137","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:ae6b7100b939d4713d6203c0162538eccb6b9fb7673d7e580bb79242feba84a1","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:e8dc657b55aeca646cf8da22b50dd53365b2d880ae89cfe3b803a322e788e396","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:aedb26de99ea67bdf0d0392ceb5f4f563742abe88d1a3e324c407fe2aebfcce7","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:79e9c7d35f2b784cb940f89e4f2b30053200b4ea4542fff0856f7cd97be06d09","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:8a9b61bb666ee14d929abc34969bcf9c745e867bf29b4bdfeca24554c5cbeeb5","evidence_path":"evidence/20260721T160343Z-771ba56a/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:ec56c453e0c51e02d42fc6693e99cbf640f307b957dedc9c56e92ed1d62d4d26","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:f9ece98160e878c3bbcd777e87f76138a1757f444fe752a8a5743d2fd3db8c09","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:c9ab29e023ba4857b5450e6a98b00b42f3c85d8870701bf93c14174738e33bf4","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:3768e6e16b724a2b1958dcf152b39e1e3725e7684201ab5e5e7903c108cb0771","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:6d73d0a17b22781bfe4dc9b37d57ac5c9216914c09c5f7f944d91c50755a84fa","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:0e6950657c0584a31c8db04bb32967cf0125c438dc4c72754dcf05ee96ca8cd1","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:60d602156577292b92becb1369183dd40d70b0775aeb16d9788fd1287214f9cb","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:0adb07725489a46448f284fa6b7159ab00b27a219a161961924efeb25f24e866","evidence_path":"evidence/20260721T160343Z-771ba56a/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:52c5d3500daf6b9bf046c40ae0448b787dda5e0f0e94535a1f7e5ac0a125b669","evidence_path":"evidence/20260721T160343Z-771ba56a/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:f41de5d17099a33468bc63a0914cd77b9539b285e1d17062748b39d66921df54","evidence_path":"evidence/20260721T160343Z-771ba56a/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:d1f198f423028135bf35507a005ca69b0596e85fe4d13ee2791c50abec74401f","evidence_path":"evidence/20260721T160343Z-771ba56a/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:fc042120cf61424911893c28723ba584a9b35d6d157a651c6344e544484fc7e3","evidence_path":"evidence/20260721T160343Z-771ba56a/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:1ff13287717f64bc34b1579c65d7acd3a8a065ec4f4d7b5ec6de83d14eededbb","evidence_path":"evidence/20260721T160343Z-771ba56a/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:16f72c2e4d0bfdf811c5a65fd1cef136f531a35929bf08bac082b419bbfb057d","evidence_path":"evidence/20260721T160343Z-771ba56a/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:cbf9c0f36ef42a6c88235329237e5115790b769ea2398777e94672d59f7bb7f2","evidence_path":"evidence/20260721T160343Z-771ba56a/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:7e3d420f98a8db548ebcaf1e7ff4cee8d849ad50aaa8502ad31d837f9db5609d","evidence_path":"evidence/20260721T160343Z-771ba56a/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:0d0c12c384a7bfc41c3e65bbe311778dead579676b5dff4901725857e5b9c22f","evidence_path":"evidence/20260721T160343Z-771ba56a/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:77fbd7550a4d270bf3925797c782d662e9f592893f0a86d8df7af78f60fe7dd2","evidence_path":"evidence/20260721T160343Z-771ba56a/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:24feb02628467259ec54d6d07dbeb1cbf6246c8b0c5d5f8293e606f993cfdeda","evidence_path":"evidence/20260721T160343Z-771ba56a/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:4cf4dfbb9966070b2f23609e630fb34505bbbc1336dc8bd99ee9b40a8e633183","evidence_path":"evidence/20260721T160343Z-771ba56a/single-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:0d09cfd1036ba520b2316466f5967df178eb1bca319010309b8460a2ed1377da","evidence_path":"evidence/20260721T160343Z-771ba56a/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:235a0b4d5b0cc7669885e53dcd2a9e205530e526b76964d686bbc7bdda86e005","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:2535fe01783e1f5cd1c68459d67f9fa388e79fbf855ffbe91541e610990d3c84","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:7c75f65b4b6482875a3fbb02aaab58f6192440cd7c967fb886b12dd2b7e1f50a","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:3b0ed9f239ec2aa78632aa93ba7f81a6d1fb3494970b73637e19f36e52e2314c","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:6f93c7beeed76337077a2242e5e7890532f9b83711d8f093fa6d64e99dde4d11","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:f0cd31342b3b05f4ec9d26a965b3cb31f7f674fd9eddc97120969f7064dd9995","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:6c62b4c38fb6a3c7b79400ee086f3f4c1bbccc8eeb92097585e19d1b0c5fe200","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:36ead9296bfe943b711a23547e3b8656d4b5fed07b817d07a1dd347ad784f46c","evidence_path":"evidence/20260721T160343Z-771ba56a/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:649969ded8c0613ec0bab97d54f9074eafb8ecb913b2261a49bb8392ac94b00d","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:980e83b5db04c5cfa9c924881fae497df57c6e6dfa6054ba4aa9b8ea7874e081","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:3ce446d56d03f191283835a85e529b2d4f0471f44f932bc111906f2c742786f4","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:4bc1afe78e942237417e82fe926197ef36bd6b11f0b9998145a38c2f5bc9d9ad","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:cc1d9c14335e37853b71d9255c9d8380e2b2c79fff9dc25d0040b7df1119de7f","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:75898d0f4e72bde98cbe51efa3b941b0a28056caa01e34c23cb86137ff1a57cf","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:7e26b8d241269fdf800d6788a10977ae238a4351233eca6e9352b30efe2c94ff","evidence_path":"evidence/20260721T160343Z-771ba56a/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":15,"timestamp":"2026-07-21T16:03:43Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"4bit","scheme":null},"revision":"8b403126fc14f14cfc99bb4cfa72ecbc129ea677","sha256":null,"source_id":"mlx-community/Qwen2.5-1.5B-Instruct-4bit","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-1.5B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-1.5B-Instruct","endpoint_id":"mlx-community/Qwen2.5-1.5B-Instruct-4bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-qwen2.5-1.5b-instruct-4bit.json","run_id":"20260721T160226Z-3d6b0b23","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:96615b7dcd3100b0b755eee927c20d5e4b59ce7af0c07c2c834709f5d6e0dfe1","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:ce5e21c265c78da1ed9f91f2e7e41776840f67408ce1be086a453bc2e2229308","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:af6fe986c7d317ca7c4915f859d862e22345ef565d964cf1c4fe25be1d2e98ae","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_order'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:914fd762e4db333a3013e41019040251e3be63fa17f949bd900857b0b4f199da","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:d33a818c14f377e23baaa74ab9436beaa2c38ec09c4270ea38e635f1d9150786","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:6b4c4e1d0e6fe806c808622389edeba8704eb4b5ac920623a282972e9a371819","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:7e4af51a8382eaa7c2c03c20ed68288c62aa1a412d84eeea00d8fe6d0d9e54d5","evidence_path":"evidence/20260721T160226Z-3d6b0b23/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:9bcd892716f9427160defdf7ee32de46c3d98e3594d08f11cd52f453f0132e07","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:f130ca01ce9b9d5b67bd28e351df341fb858987e3a3700fb2a1ab6a67028ee14","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:b940aac00432181d417b0182f7f502d34723d83cd2609643af61e09cd1915cce","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:78791db5de0bd59879d8566c93bdf3665b4806437e0f76818fd04f3fcc550b5e","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:5a1f4786543a6a03f59cdac4b28656a1b1a2dcc2c2002d0359137fe0f48d9763","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:c57b1f25bf1ea6137b6abba73f6cb42e70907d0212e8d9b05111959a8e9231fd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:e8eb639d7cce5a5e9f6d34dd577353a9da02c4f7a38fbb15be6b5f83f86416de","evidence_path":"evidence/20260721T160226Z-3d6b0b23/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:637d7b5878f636735dc3dc43bec5b49067868483bcd9b3dde47597ebd67f7831","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:61ebe4797b70d84998243517232189c64325e8f6720a8f29c1b3644e62ebf614","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:cab6d938edbd76da2e9363909b6983c0913cae9ea216a6c6a621080370b419a7","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:00b08aca08588ef8f8e9af35a5b809b8f552b938432e1a322298f5e6963ba21f","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:ef3e1e457747cdd1a9413f96e3c8572c8b4f7e445c6250eb5740c78ad3cd4590","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:15651f27e6e3a0e35e622eda6388a5ca9020ab8ef18cfd7d25a9b02dc9ca04bd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:b09f1dccfda7aeee680fc1fb77842033db4f50cf13c28abcde232684b06cdea9","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:776b52e9822f7b9dd3fd4d660aca4974b2f25b9564674842127b9167e0f908dd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:380c9d683fb2dd7ffff12a68f0cfc266312a3b71f38e8db24d4301303059ab73","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:79b74737cd5928388160c76ea9b0e559078dfa1cfaf374aeec2f658e50fa86a7","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:ccebecd42d2ad7f97091e82e9c39aa50a27e6e3cec68ea34e3e26693804d8ecd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:0e22ec4850db4d543472193d0e21dd9e83fc6632515093b27c07169d65a3cdc2","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:c962a4f8b25b3cf5b1337a96cf518ae74a1d9dd540c741c4e93ff206b4e24837","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:ab698b1c975d7f2aa016a1a3cc3a61bac5715ce96cc1fe9868b6987abd63f27c","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:9e6aa3f707e98c6e7acb9d2d480e93c3f839a699434a6f63291d308d717e40cb","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:4f1a9c5fceb1e0fddd3d25412df4f8ee439dc2794809e678b5f3d9d2eeceecac","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'prompt'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:f010c69653b103f57cef01be95e4cb7039e84df83fe16f0a5efb2dbfd9aefe46","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:c7805e29c394236733926e21d1d07a6b4ea9de4b55eb2a4b00cd7fea540633b0","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:8a5663502397f946477d7d8f1e2d3187468dd3e80b3209b08089bae71d47d336","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:f21668fcce2f7531109d902e4c069fa2c0262ee3066d853783919e9a1fab3abd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:8b1daf34b2c3e6df3634007347cd1029b52f44fb75309485571ec0eaea28fa0f","evidence_path":"evidence/20260721T160226Z-3d6b0b23/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:47842ecdf31c8ee44ebe71578effedf504d03b9afd5c72f20532579348dcc816","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:4226464b992504e426130177125b19aaa281a90979d18a1ce857fef3808dbffe","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:77038546a14742c56d493170984e4b6b8dd1ea58316f853a544fbbc65289510e","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:9109e475a1235a094eb97f3a26c65095f7a62bf2350837d6a3e2b0e4ed346479","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:3e77ba3448fd694051bc0325e469a4c4f96b5a7854669ed49d1275ee8f44c0cf","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:2025e56b1213e487b5b48bfe9bbbde78fece7e07868ba1cd9ec30baccd4e2d3e","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:55ac94f5081b98bb488cf90e326b2eb7cd575c4e315d57cb600fcfd72ea8405c","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:fd2d61476069feb1ce9b52e757ff37978ead6591ee272748db7e24458bca837f","evidence_path":"evidence/20260721T160226Z-3d6b0b23/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:328481ff952a86502d6eac3dd2e5c5eecacdac22e3d345331ca0c7576e229c45","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:6670a59f14fbe8e83ff4cc337784765295d5f2d4dabdd07c1755245a08c8997b","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:89b4b444cf6f9586ac680be8180f4243191a710848c9cf3e5beeeff74eaeaf6e","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:88b3c8f81e074d20ae5d7a936de20ca820eb1a7b14119f898c6dd1f28484e74b","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:a230999200d2b0085e689414ade924147b9b687998e0ea37ab31683056726a91","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:482d4b60f3dd3a01c10a5235a42251dd5789878ed4ab7d68f702750d06db4842","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:651d39e49d03b5f0047e5b2257f56a1e0612f868f49f8a605aeaeb35fc8e56fd","evidence_path":"evidence/20260721T160226Z-3d6b0b23/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":16,"timestamp":"2026-07-21T16:02:26Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"4bit","scheme":null},"revision":"c26a38f6a37d0a51b4e9a1eb3026530fa35d9fed","sha256":null,"source_id":"mlx-community/Qwen2.5-7B-Instruct-4bit","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-7B-Instruct","endpoint_id":"mlx-community/Qwen2.5-7B-Instruct-4bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-qwen2.5-7b-instruct-4bit.json","run_id":"20260721T160019Z-4145246d","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:d4336663a0db61feaaaf0420143faf14c493d711beaf2a9660fe7ca332bb91ae","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:aacb6cde2d9ed4c1ba0647252df1782c89ebf8514c60406e48b7142296f47558","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:71412c39f3563aec600b920a2e32e18be714efa7255facfc04dd0d3e29ff0b30","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:449da0d4540c879633834f19f3995701fb5babbab472a9495f7e681bf2be842b","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:732090f0ec935c92f14683e9969b6d952ffef2be2781b28e99f5655e2973acf1","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:3686c9f69dc2f7d02a729b26182b225116abf3969f5ec36a31db36629bd8817a","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:f07f9c5ea3bcdb3d87f1cf0e0e782e748b6bdc3398c63604829a213c7287285b","evidence_path":"evidence/20260721T160019Z-4145246d/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:6ab1457dfc9b0684450e0b0faf6265c263b09e960ac764b138d28a81b35cc32c","evidence_path":"evidence/20260721T160019Z-4145246d/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:4daca6f7dc5a58077ff48169432c9a417539a74bb52864b69e2c8b4c8c52ce40","evidence_path":"evidence/20260721T160019Z-4145246d/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:09cd6777bafe024f7b9775360ec0b84c7203b0f65e04b802b2e5c2fd95a0b309","evidence_path":"evidence/20260721T160019Z-4145246d/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:95d889641f83161671f477844e676e923ae72a0914701e8452ac49d1be94210b","evidence_path":"evidence/20260721T160019Z-4145246d/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:85788c8f4080cea10dd785dc67ded4720ee8bf040a144a821b8e936f44737b3c","evidence_path":"evidence/20260721T160019Z-4145246d/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:25b906939c4faebf1a6643d66b428bb5ab6dab05b22b1cb97c6a0359fcfb5013","evidence_path":"evidence/20260721T160019Z-4145246d/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:a18e3e1043e065edaae39a3f5071143065e6b4ae73aa10a4c2d4624d48eab98b","evidence_path":"evidence/20260721T160019Z-4145246d/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:250c96e27cb64098f9ac6de28bc54be81473c19e6bafcfb3556426334c03b7e6","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:a8ac159abe9cbd133ee88ea773f2e0ca4d34b1d33e10fe0316324a3f4db731c3","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:503a06ebe7eb6648b221057181bec316ab2d8529f274cd648d83394cb145e96c","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:94396ed20bb5a1e8ae73691a4f015ac773467de368061a0850e17c1d3988d708","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:bb4629faf8f6a8da01aa1bb1ac830005fb65b38ff2580a9b9ed6d469df970799","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:a21f112be26ac4e23b9424aeac107505f686d74a26e7010e3bbf506f01676063","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:3a29b68d0c2b246c3315c46c385eceea9708b417b033d02c1f6a05be15be5463","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:cdc09c8901671dd9bb89b6671518c8299674b8b8dc398d80d9a5a1a4b860106f","evidence_path":"evidence/20260721T160019Z-4145246d/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:b5958f1b39d2a023bf3b8789fe09f79814dbcaf08d911a133a8ebf719e00ab04","evidence_path":"evidence/20260721T160019Z-4145246d/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:ef935b76ecb052f75d52c0add7f0c8920780f255c10b4de2fe748a688a7e0fe7","evidence_path":"evidence/20260721T160019Z-4145246d/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:1450a375538316e9d08d9111a0b5e054c28ed3897d75ae21caf61133459de6ec","evidence_path":"evidence/20260721T160019Z-4145246d/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:2d3d47c2968f0f5e7b48d0772e35007be58fd48c548f23cd6a0dd9c46632ec6a","evidence_path":"evidence/20260721T160019Z-4145246d/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:4487981af21a7f83365d51b46eeb8a891beeea59d980503aa58322b4e27c8f68","evidence_path":"evidence/20260721T160019Z-4145246d/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:487c83a306ab696166fdf0130ca3dca949fff1b4dba086b13103fb9d97efa5f5","evidence_path":"evidence/20260721T160019Z-4145246d/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:800f43ad92a89d65ab9ab2393586e749b84c5e6d712896ab167b55ee8e10661e","evidence_path":"evidence/20260721T160019Z-4145246d/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:4367d3599e82cec3c44817c65c6de3a887c0b03d6c1a2c29cac97e600120ae56","evidence_path":"evidence/20260721T160019Z-4145246d/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:c2f588c86975f9b32843c7ce71dc7dd87b63f51469b7cb90941769f0cc037acb","evidence_path":"evidence/20260721T160019Z-4145246d/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:4430857a8667855c2e4f14df906071798237dfa1925029e795655dffd23ef84e","evidence_path":"evidence/20260721T160019Z-4145246d/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:9cb19537eb1ec76061238a3c8c5cc180740f4a8e6d61342a9207795d6ffc88c4","evidence_path":"evidence/20260721T160019Z-4145246d/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:886763fc1b9630261014e41d9d9a3e1d44b454b173e7a9017379d754118d9dee","evidence_path":"evidence/20260721T160019Z-4145246d/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:c365df4cde5af737720893faeb26d2ab0495a0a5a2c6fdb40d15f468bdab57b2","evidence_path":"evidence/20260721T160019Z-4145246d/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:546c71b9b1a8c12c5d7595700f89ee996f474425907e5479bcf201335d03a468","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:c72488f65b01d706c89366b5d0eca7af30f85931d73f16cdfb9420a047819211","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:ce8081c8eb8979f4d912194a1602a6488c24106567a3f407734da198902c02e7","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:b0fb9e485cbfd3cc160b3774d7d9103018b9af58a0e1be2455f32c343588a325","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:ee46db91828a214143b4d8fa377046bb6bf583c15caf3cc583797b82dc44f814","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:a1a5ecbad52ba370b8b197b6a4215771cac8e44f8a9e1de425b13c89f8272847","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:ffa48ed3f1a156a54d1ec0c6626ddae05b7f03f0c34dd0eae639dd80c26b0d55","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:bd17e3c0f76cf9f330a35fe91f9f8d30e27ae6044a77d04ce99f91168029f045","evidence_path":"evidence/20260721T160019Z-4145246d/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:a85fdfb06414c36e8e4936e2beba1975550b3dcb61d19b4b7ff25e56d4e54280","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:25f6dad8576b8216c7d451b9db3edab17c9a75918fb63ce42f1ae4726a8b5ded","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:e1e4f5429c7a5308cf753f2fe34d756923a3d8b9513589dae0af64c92c28fea2","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:a0b341cbe3cddc00c1d921a9a6210ab23098cb565a87115ad8de65a7b7601a0d","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:b519fb3e09db8a87b3bd6ae73b3b67a61e1002f113f47b8b17d2046d7c036a0b","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:f91e73176c62603d3940c1d5b10d041a60b9ea46878e9f33e7a8b0eaf46f63da","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:9202d9a9cd6ee7f1dd4b9f45641220011a3346d46b21893a7a5e1fe4f7ef233d","evidence_path":"evidence/20260721T160019Z-4145246d/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":17,"timestamp":"2026-07-21T16:00:19Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"mlx","quantization":{"bits":null,"label":"8bit","scheme":null},"revision":"df11cb390bd674c24a9ddedf19f05bc09f6e0148","sha256":null,"source_id":"mlx-community/Qwen2.5-7B-Instruct-8bit","source_kind":"huggingface"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"Qwen/Qwen2.5-7B-Instruct","endpoint_id":"mlx-community/Qwen2.5-7B-Instruct-8bit","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"mlx_lm-qwen2.5-7b-instruct-8bit.json","run_id":"20260721T160109Z-829d726e","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:cbdf2ba65619e09ed5932c04b36496360a3f6e8de1a8c5576ed0847a4f09a8bd","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-calendar-followup.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:814f3daf6fb26b4a24c4a3af80ed1cb6493d70f79c19d95a21466310afd8be97","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-flight-booking.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:a453cc3c5b16fba896fdbdf30a269a8c2d3ee4219a576249a034d2c9813069a7","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-inventory-order.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:0b92a3fdb8af1438777822221b049b32392f88c77994108c768aee57b76d1600","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:da70f359146b734c2df0afa40f4d3e6480974add698539f575502d5882d23d93","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-route.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:2875eb0e6dff98dc6cf8393dfcb98dcc350dfd1259ad338e810f5d7cdfea7de9","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:525c0f7a682daf9605a126ef658992499969ea6ed065806db17eec606af2568d","evidence_path":"evidence/20260721T160109Z-829d726e/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:43655b90f32318d6a8daaa5cde6ea842166e15b8ef97bd588ecccbd50657c6ed","evidence_path":"evidence/20260721T160109Z-829d726e/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:90e8bd085b5c9bed61ca0ff52a747c66c0a9952ae217c2019088ae3d4f2d429f","evidence_path":"evidence/20260721T160109Z-829d726e/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:7090dcea8ea3d8adafbab682dae0ef63362e58cc27bb6d811e322d106e25b5d1","evidence_path":"evidence/20260721T160109Z-829d726e/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:e0919a0f5fa8da1a1cc95d1c89a3a69e3e5dfa34a719647042c917758edbbacf","evidence_path":"evidence/20260721T160109Z-829d726e/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:b37bcba5bb6f4a0dd3819ee5a917ea250761da4772579847df6e47631ceb8da1","evidence_path":"evidence/20260721T160109Z-829d726e/negative-long-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:01eab3c5b58a7a0943a3cc4328017468f3c2ee9796d313f369af04596b3a4ecc","evidence_path":"evidence/20260721T160109Z-829d726e/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:9f678eea6038ffbcee170af299569391b6605b99a6dc0a6e77f1310fb9a038f6","evidence_path":"evidence/20260721T160109Z-829d726e/negative-unicode-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:ee1423cfa131474ab48887c719988ec2e25ad1b361b5f5e1e20c9dcf5db55bd4","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:7bedd717cf205866100c3128a4a52631e7d769316937c78aaa2f5eb58097e87f","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:6f9ff270b6a085c287a742349bdfc39997c2a11ab19720d4f66bb71cc1325a18","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:5f3da9a505e4b0b028b98cf280bc30c1288877fe9ab710843a731f647c8f4170","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:8d1c5c06d2509ba9329adf62e587138220a680848f4cb6de890963d5b00ec6ea","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:6ecc5b2adfc8c7a44d5c5b2f631aac41da657ea301ce98e1f53d989d555676e4","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:965e4c9d58e613e6a0e30552c3f5adb2a08af62f86a1ebda314ac65ff01f95b8","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:dbb81741acfbd8e01a729b2fcb93866349a5963bf9e8fee3ec4a085a79636d55","evidence_path":"evidence/20260721T160109Z-829d726e/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:89e16b1a389039bf0e00e253fec4857cf8c1eab931db0297c7e87c21b73b9fc8","evidence_path":"evidence/20260721T160109Z-829d726e/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:8dfee586df801010d51569140c7590acd52431b4348566d6141951a3bccefd73","evidence_path":"evidence/20260721T160109Z-829d726e/single-boolean-flag.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:1638b94fce4d66d3ac3f32ec5825bd822eedeae7d8ca6ae624bdadea968abbf4","evidence_path":"evidence/20260721T160109Z-829d726e/single-date-range.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:0daf774166735a48bf24613ed2c9f6edf8669487e3d3ec74d68a0bc41e0a0457","evidence_path":"evidence/20260721T160109Z-829d726e/single-decimal-price.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:71e59905f8b2b8e7d68b2eaa3bed3096ffab73739f34e619992230ff37e9fb21","evidence_path":"evidence/20260721T160109Z-829d726e/single-empty-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:97c7d04dd40669c8df14a99cb0802cb896c9c6d6142ee2295aeae9d49db074fe","evidence_path":"evidence/20260721T160109Z-829d726e/single-enum-format.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:3ecfe52d20141a98e70f3a95bddd4d976234e1b3e5471918db074ab40c6385be","evidence_path":"evidence/20260721T160109Z-829d726e/single-integer-limit.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:107059d75205664f4df9b6fde59d557c8f0f6cc74d06eb7a8fae001e4a7b6d73","evidence_path":"evidence/20260721T160109Z-829d726e/single-nested-options.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:94bc0005b1ae300bcecd189d47b3c7e13b8090606f0c4561a4f2613789a2d08a","evidence_path":"evidence/20260721T160109Z-829d726e/single-optional-omitted.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:cd27219aab76e72c3efdb5b871676541b8c70608f6d9897579c80148832a997a","evidence_path":"evidence/20260721T160109Z-829d726e/single-optional-present.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:8c4d8a7ad44ab7e48aa30d1b1e2df2764911bec204be802549d5625920b15929","evidence_path":"evidence/20260721T160109Z-829d726e/single-string-list.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:04fce31c30241f99fc2f97f5cadb50389d48a4c3ea803728f313accfc5b884c5","evidence_path":"evidence/20260721T160109Z-829d726e/single-weather.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:b20cbc8e25df8435c2982c28919d7c4c44ad5c8395a48bef5f8e08fbdc02a47c","evidence_path":"evidence/20260721T160109Z-829d726e/single-zero-and-negative.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:54a353508840e9b1a28fd6dc8d9632ca736b3180ac5cbe5d705280d99827c8d2","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-array-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:cd159dd041e66b2a84d4d9d30deb831383fa0706e63b3e419c8569ca2f8fb670","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-boolean-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:c05e3ebb7b3b252e50e7f43cb9055b0ba86ad4e3540327443bafb273ef3d93a2","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-empty-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:91a20ee193d2d93689c11836f18d4eb1a1b7d73d19ab401ad4dc17aa4c280b71","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-enum-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:02ca46e059e526fb91f0745be08bd29319c8be704368a1a0400d95165364a655","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-nested-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:054c6315a56133c046bd3bd28f72c3adf15245537751354914ed34e8111cf70c","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:ea815f898f6ca3da86bb6d7704e4eff282557260f21280876da705a865d60799","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:c8743b2876cabc4c247278500b8e2bdca0fb7b6f7356044f409bd924e7fb95f3","evidence_path":"evidence/20260721T160109Z-829d726e/streaming-weather.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:a0680416e4adc710a8bc91fc0c00ad6daf31f4a78cb49d4a973f86b74f4d5659","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-auto-call.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:31eb6369ada0b6b1d496f2174c1a7061e627ac0cf9f4a5e96dce4750f8948d18","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:f9c00f89faeec06dc9c28ff70de3dd0a7ec9f6804bc10bf6285d4357615ec230","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-named.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:054056495893b232f264133a7acf28bf43b542b48d20f106f9edb13ad6614477","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-named-empty.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:c284b9050c663e6fe71ccadcbe35b5fd99e9ec1cf3690a3cdf833cb89d2198f9","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:7f0a3f7660ff1f58e1d407d339edbb2f348e11216a8b50526ae552845164f842","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-required.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:013ed8e6697b0d5b560f6edcdeca746d0ce5591fc597462795508edf7e07db58","evidence_path":"evidence/20260721T160109Z-829d726e/tool-choice-required-empty.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"recorded","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"mlx_lm","quirk_flags":["unconstrained_post_hoc_parse"],"recorded_decode_mode":"unconstrained_post_hoc","reported_version":null},"stack_index":18,"timestamp":"2026-07-21T16:01:09Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:f4031aab637d1ffa37b42570452ae0e4fad0314754d17ded67322e4b95836f8a","sha256":"sha256:e8ad13eff07a78d89926e9e8b882317d082ef5bf9768ad7b50fcdbbcd63748de","source_id":"gemma3:12b","source_kind":"ollama"},"canonical_id":null,"corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"gemma3:12b","endpoint_id":"gemma3:12b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"unresolved","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-gemma3-12b.json","run_id":"20260720T181615Z-5a462971","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:f1c30c0566d7b9409de8706633fbad99ddc50319e1b7469ba824af2cfa9b4079","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:ac0b910b95b409aa2d68c155e572ab4a157c3695bf826138cbbd5034d63fe23a","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:bfab25eeb9a474957779a5f14eb73bd4d945bc2992ea28d36b902f526b4f6ca1","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:f217a6be18eee271a786555a8e7095bef7129680eb0c011effbfeca3d83fc12f","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:add0b25a9ffc86c9ecce9f2ad0d86c359592e466f25c585460e8447393b029a3","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:289eecd09feae0d79355a1d2cfcfabb92392b721d8562fef0e27be2854363b22","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:2fbe8005e1d0a715a979e30172de95acce4c588df99223dba9beab51293feddc","evidence_path":"evidence/20260720T181615Z-5a462971/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:519bae1ce349d4c088c39e5a4b2c7daee09cfb99547009216bb8d34e1e78d4d7","evidence_path":"evidence/20260720T181615Z-5a462971/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:7affba4944ef7973ba7223dca5042b5719e545fd914a48d3019b9c90e052bae7","evidence_path":"evidence/20260720T181615Z-5a462971/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:aefc303229b715b5928a5571370d4e8f84978cf6e2f1fd51d4a834614cf6dbea","evidence_path":"evidence/20260720T181615Z-5a462971/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:ac6f322055fd2fee7c52a991241370844132adcadb240b26c104cbdc0f2ac6f8","evidence_path":"evidence/20260720T181615Z-5a462971/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:3b26add1bf7bfa0f93414322fee7cd476d891656385b6399a2283be548329fbe","evidence_path":"evidence/20260720T181615Z-5a462971/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:a0284740a0959afa47080f8bf3cc4aa7f366ac22e13cfaf2f019f38436e0f2fb","evidence_path":"evidence/20260720T181615Z-5a462971/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:3353a0e2479f59713c6ff90bdb685dfe6572a50598aaaf5a83ee69db0cf88a4a","evidence_path":"evidence/20260720T181615Z-5a462971/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:03e8640fe71e9e1bc17b626a9792eb6d868beccc2db55f820bb33bf35058e6e5","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:8c795bca03a104abdefc9444fd7dfa0bc6302c92aef5c0a5a9eacb5e93ea2e33","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:58c2848ea145aa12927cab3d67bd23c5772911e233e0b05c5a8adb36859d2427","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:a8a3228a2a07f89e9dbdc39466fdeef0b60c2e337c8bf6c474d1716b340e8452","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:3def2b0e424bed3a5d1703180aa454055e5727d6bd7c41bcd2f7ed9392d5b200","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:c8cb49f2b40d4a8eb586a96b59e2405d61cbfb58124e56fe8ae25fd748f43391","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:fcd6a778370712459af308f4ffdf120aae4017cfe5e30e900772f676a840628c","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:ec4d04f47097e0daf507342d81b20aac80ee247eefb26dcc22571c10104ded73","evidence_path":"evidence/20260720T181615Z-5a462971/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:274d58b2da44e564b9e748c6ea7760d22fb71ab72df0eb220da648ef8ae0ec29","evidence_path":"evidence/20260720T181615Z-5a462971/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:afa1a69557119d871fb8094c459ed5fb26814a8a42a79d6ef2c3959df55d4e94","evidence_path":"evidence/20260720T181615Z-5a462971/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:6d42dc1798a6baf34763a59222cad311ed0a83da3c4d844be4cb02d2c1d3b08a","evidence_path":"evidence/20260720T181615Z-5a462971/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:f1c1a8732d6a77eefc2bfe62ff340e9ce4559967d3ce73db880740c1686ccd5d","evidence_path":"evidence/20260720T181615Z-5a462971/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:1e41b1799f55d5f10a620ecdbb501a237b9ad945570ffe116598b1b8fb0d7568","evidence_path":"evidence/20260720T181615Z-5a462971/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:30ec93a6e03511c432c06341d7cc27bdef752282d5f48b321285b1dcb35717dc","evidence_path":"evidence/20260720T181615Z-5a462971/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:e9982f5dc604a3ff6b5e04559a717c1b78a1e408f3b550a4736c9a52e55a06cd","evidence_path":"evidence/20260720T181615Z-5a462971/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:e38835c8b3fc3a1de285f2ab3c2ea2f80c216fe9e3875865da3689d52fd60ff8","evidence_path":"evidence/20260720T181615Z-5a462971/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:519dc9554ce454013170fc1a585c56798c6c5307ed41f340154c138b37abb90c","evidence_path":"evidence/20260720T181615Z-5a462971/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:b24731be9c34a21ecc90b854d0559f6c063a7b9852ce562e8b9b1bc4939d322e","evidence_path":"evidence/20260720T181615Z-5a462971/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:85e4f8cc36476f41264ebda360a1be7aecb6ec564bd3790b99e8ff1bb94f41bf","evidence_path":"evidence/20260720T181615Z-5a462971/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:3592ad13701d43a71840b2a0e7c32ce0fd4f1e25788f6ee320f5711b115fc4f9","evidence_path":"evidence/20260720T181615Z-5a462971/single-weather.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:10e5f8864be401d240294abd98f404dcd3e9ee87ce44453af2aa340337f23860","evidence_path":"evidence/20260720T181615Z-5a462971/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:4d6c5d088bf4526639d72df70b0d30a9dbdd8b1fe30a7b4770ab0f0e82202c17","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:c35ffdb7cf4af72150ca72cc2d2e4e064f1b3eb5d5a9e6b2acf6ddd0d4c300ae","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:26962caa8c9eda69ee7bb5afc50e40a2c8ef6658d10325e92e0a497734350949","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:1d4bc65fdb1f24b1208ce8fbf0854419460f90f44a156b45d547e9385457735a","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:86357a875cd7ae71bb6636e524fcb7d51b5a96ed7a2da00e5839cdd31488dfcd","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:149b6e0a182df47f81b9c703980c62d2506467782032c021d6fe353db0c9a0ff","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:30ef5839b0b149acb17548a43331b1f43397e4a8cceb8380ff9fbdbe5c8323e7","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:5206174cac4b4d6b8a6a9c64a1b221557e39aaefd6d35169b377b79ac99e087a","evidence_path":"evidence/20260720T181615Z-5a462971/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:c39255b5b2edc0e3c31b8124b6840b5b812688404f1095c14b26588d4592e674","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:87420925423a328af0293a8862094ca09080b0430deb1949b4a41c3ef811af6e","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:a788cb4fcff2f9174202d84aa924bd43590682fd842e1be5b5e3f7865e0d41f6","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:cbeb5999e1725db672e7b7d3bc95633553203da24301eba6a1543980572050d8","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:b89f1dfdbd2d3f9e96a071a8a8120ae8d0d55e818f45ab44af7e29b73edc2d94","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:703c8d8f55e18616a35bc942223cbbbc8c3449a2c52569bbff043d73705541bd","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:d2df599bcfb2a935d694a0e56ba2716fc5131b16234a0e9d42d289bee4132dba","evidence_path":"evidence/20260720T181615Z-5a462971/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"error"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":19,"timestamp":"2026-07-20T18:16:15Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:a2af6cc3eb7fa8be8504abaf9b04e88f17a119ec3f04a3addf55f92841195f5a","sha256":"sha256:aeda25e63ebd698fab8638ffb778e68bed908b960d39d0becc650fa981609d25","source_id":"gemma3:4b","source_kind":"ollama"},"canonical_id":null,"corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"gemma3:4b","endpoint_id":"gemma3:4b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"unresolved","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-gemma3-4b.json","run_id":"20260720T181603Z-396e39a4","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:9ce4e1f5f3d53bf47c71934d8bedb530383be6b36d1d140a597095beeefcd3f9","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:17dd2d94b963706e0f9c67961fc288896a73b67e0c1c2cf1a453b384ec5fc7e5","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:41974be9d9080dce953d6bcca33dfc3a925eac211b15191d34190aa0e0f44d28","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:078ab2786b0ef9bd0fddd57ee41603af769a9f4bbb5536d5702a25f265f04910","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:d1cc3e0fbcdf1514ca05f2a8c84d9ffac528a2687cc256075152af7a79238aef","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:ac81f6a6b077f797feaa2801125bc47f6ce086ceae0afd24bb9911c9ec83a2da","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"error"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:098155aacafee7b125ffb971373d35960890a77c385f511ebae19f7b3a2fd5bc","evidence_path":"evidence/20260720T181603Z-396e39a4/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: server returned HTTP 400 Bad Request","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:b30a802474e7391e35c556c611524f94a816db97de3874ae72b872f576ff3130","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:309593985c6f17d0a9d171594819ed9ad6df82ee7b6f55b7084b3e69fcc610f1","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:a6ab8c2b245757fba36d59230b43406ee6bdecc25c18d81d158aa42950ffc181","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:afaf480fe12559806b283808c9011c5e8b000ef749293a327c2a1c66cf48a178","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:2e6af3720e56d09317dafca5d94f84a6c004fd2dd4187cd99959a10cab1739c9","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:f631ecbb3c0e086fcb0885d2cca169f8b27c7334d9f3cf8880d738f71385a51e","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"error"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:146de58e3e30c219cb3bd60c7bc39036e6d81d7d32a104e0d9460a96b30264f2","evidence_path":"evidence/20260720T181603Z-396e39a4/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:d27eef042f4b9ac48ae1fcb861eee4c926a8583fdbda4a3a7eee18f9a4f74d57","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:fa05d9eba9afc01e938e2dd2e029bc340bf29c10ae02f34607582d0335f51afa","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:e4e784175fbc59ede69b8335e8c8a76beb2a65834fecfd04f2cd6cf95c55d43b","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:7d00cecdff96a70318035e71b0bbb4a84aae25ad6e8c12ac76e6be8d4bca3ad2","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:9c466826634bf74a2fbdd7d9b734bbd5bc1a804233937d76c8bec27e1bf76ada","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:1898bc323cc884152761d3bb68c124383fa874f5b725baa4abb86892b415c677","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:d7f300af044156ac0f145ced3eadef26284b106bafb4dc0c4e55eca24a35c4af","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"error"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:2eb9db70af4eecd263f5e2c7928e2899cee62af9dec04ed7affcb0379f4fedfe","evidence_path":"evidence/20260720T181603Z-396e39a4/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:b5cc8194bd9e38013814a795c25d3e96857ead7977bfa4f7a4d6ae054c9f895b","evidence_path":"evidence/20260720T181603Z-396e39a4/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:c512ae007e925015416c8281b3387ec7d07ec02a9f77ea83ac128672811976a3","evidence_path":"evidence/20260720T181603Z-396e39a4/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:ccd2fa87ef63328ff3b23f1b2847320a817b6b55350f94ed3f958b08226f9c2a","evidence_path":"evidence/20260720T181603Z-396e39a4/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:95122158fb74a4ed45e6ac00c67e6987b901d26422e2a5a3df3ad0d373ba7c2a","evidence_path":"evidence/20260720T181603Z-396e39a4/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:aebe91cda9f65b7daaa4a50bfcb7edff4eaf6e368f071794cb7b1f5c3b009e29","evidence_path":"evidence/20260720T181603Z-396e39a4/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:a4885836bcc4b7b9ff78729a45a8b97ae650cccfc2fe0ea37bae50f33ea7786b","evidence_path":"evidence/20260720T181603Z-396e39a4/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:2fd6281aead9e1f73072386cb6bc3b5584e24afa421618689eecff22c4cef408","evidence_path":"evidence/20260720T181603Z-396e39a4/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:15c62490c4f41ef24880fc5702bd69b7b4d185338af374709495ad14da2ed8ce","evidence_path":"evidence/20260720T181603Z-396e39a4/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:fd3896b8b6cb25cb219bf985512985d062f85fc44f16d21aba26966643d18151","evidence_path":"evidence/20260720T181603Z-396e39a4/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:7de6826441ca92eaede10b5cb6e85ffba160045ec7f5772410bccb1bde82d11c","evidence_path":"evidence/20260720T181603Z-396e39a4/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:79d021004038db739b17d3fd3d68dd3d53e094621fd910a5e906f6e74bb8300a","evidence_path":"evidence/20260720T181603Z-396e39a4/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:34c181e0e8179d5ff471580429158334717099e644339ed3f0eb83be39608d82","evidence_path":"evidence/20260720T181603Z-396e39a4/single-weather.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"error"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:1910d2cc5f753a4fbebf0979dec26b3be6063048a8279bb0abd4e171983fcb2e","evidence_path":"evidence/20260720T181603Z-396e39a4/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:140f15ab228eb19ae0dc09a3bb62c4c09ef92058849390748d29df735f02099c","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:5050bae18f03b2030e39d7761b02990f876894a2155a83715d2c74064fd1e420","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:cdb6f36bebf4c20589a23d844757332af6362d4f5703be0be44bbc3ed1c13778","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:b2f6760238c0bac9ef2dd42b5838163627beb058a96f60c664721f7bbe876363","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:15854f02381e1e0b370af866799c674c744488ad8c36806f652713746a990d77","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:546647174e7703bb871238a11fad8ecfd690fcd86c259ccfc15438db0ce2646f","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:0addfc14e446d865087cde47b17c374421e103bd0004f43bd14a7523cfcf2269","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"error"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:01da0df21f199013f1db2142964b8a36b0f64d6d5955bfe1100f0a2116593a7a","evidence_path":"evidence/20260720T181603Z-396e39a4/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:a3af311b9499880c2297615e88799457cde95a18b1562e52fac3a3151bbaad38","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:ee79c50c482c2c28dba60ba2ea95c3fc877390d7a50088dfe1a56680ba62a2c0","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:548683af2611ce2e74cf1a9a6cd3166e125aa4479dab96080e6de77545786f08","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:4d0e0e880e89138258f4430d13562eae93dd801d23ade77c1d557380ca73a9f6","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:3458dee0df44f9eab22ca93cdb1d3462800d7b68fcfe4a323a636c2c7aee1fac","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:b0d442516c371ddac9943162d34a822a706d12a7fb9908cdcce24ab8f064dddf","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"error"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:c4c39ae100fc9b8d3bfd5458d0b36bae7469501f4693f4ca417f3d0a455940d0","evidence_path":"evidence/20260720T181603Z-396e39a4/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"server returned HTTP 400 Bad Request","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"error"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":20,"timestamp":"2026-07-20T18:16:03Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:34d3be74ec547c3e1969a9ea125e1869f052f7cfd3d0bbbb5aa5e42ec033108c","sha256":"sha256:44d19d212d76a6f3fc442e8411fdb44ea6b67ceccfb00be4b4345c9a4cf813e8","source_id":"granite3.1-dense:8b","source_kind":"ollama"},"canonical_id":"ibm-granite/granite-3.1-8b-instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"granite3.1-dense:8b","endpoint_id":"granite3.1-dense:8b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-granite3.1-dense-8b.json","run_id":"20260720T181418Z-d3446dd1","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:e0074733b82122c90dd791a6bd7bb47d2201f82ba5e93db718c83178e98ade9f","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-calendar-followup.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:60ea7a99c0db260316057a46e1acf8ddc0418ee45aabe2737837b5b79db2ddd1","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-flight-booking.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:01d61e7ac809ebbbef9c1118282b8ec7ed37eac1a2ee11111e526597e5ad8850","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-inventory-order.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:e218155f8063bb2a6845eb8e9e6146e57278998a2b1eb118d9259e48965f7e44","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-parallel-followup.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:3a2751e25f8bf98f1a7fbd1f3248bbeda32f4e9a81422f339f5f3afd3bff3d1a","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-route.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:e8aa41bf902a81205a7578080df48035914c094db802da41e2f339705dc9d8e6","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-ticket-update.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:fb1e488da92450693c545da0743bfc5de8363a95f88da8f4f140f52d5d7be36b","evidence_path":"evidence/20260720T181418Z-d3446dd1/multi-turn-two-hop-chain.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:6cb3bd75d1cd1891b0c73c393add68e4041742b7123c8c8027c8fb9634a43f8e","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:c6b9e41d4d099f1ba4f540ebfd0c1c1c389cfedc8317b0e8181e3ad1d107ee2c","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:ce7e3e06c2145bad454d3a9d9165a789da6228e85f39bae48c64c2533a116717","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:73181c0446f7aae1843fa3d760b5cf39b6cc917470d5537feca2e16fd41ba979","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:fe7eeebd776a9146489d38895cbd7a317dca802f34791f12d4b1e07e75f6ba28","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-long-argument.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:bedf97ef373e64053025a79752f37ae80efd1f748b34ffff3fe30f5ddd9c8126","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:81d8e9cacd6c33b59031d441188691bb7b713179334a09da91a78a033f4ecfa3","evidence_path":"evidence/20260720T181418Z-d3446dd1/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:c77892df9985d4ea10c85c1ce975fbd2c186771a2468e16fd5bf33ff7c2b4375","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-city-time.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:5b074dd6cf6d722cebe5b6445524164603784f480e871029754796b985e79885","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-different-tools.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:cbce658821a6a915c8104a34d764c7eff324a8f9e2e99c4be7704f56f944dfd5","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-three-cities.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:63938c7ec62897f4cf1aa9baf28a3861b34e8f73d74b339b9ca3a23e45b372d8","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-three-different-tools.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:16098036c8f5ee7ada7995e383d8120f842ff264284634c14085bc626ffc9ba6","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-two-calculations.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:2acae447ae56d546c96b07534ba3109f76da2b29815c1847a4e040456386410f","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-two-multiple-args.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:ee069b235509732eae50138ddc34428fc2866c5ce2fc83cc6d547f2a7f3877d3","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-two-searches.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:693ba9ec27b961893733c0bd698ad26587bf2859a299632881af4eb785758a14","evidence_path":"evidence/20260720T181418Z-d3446dd1/parallel-weather-and-time.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:5e84576e0aee486da0ae4e22d3f9066a2facaa94951720aa61596eface098c1a","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:ee0cf2d0a2af6a32465642ab4b81154dc45069eadd09ca3162558c6ca3535823","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-boolean-flag.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:3a2775f0ba59b74a51cd11566e2a5f25518eabae77020320d584be6ba9c6055c","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-date-range.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:4e552cc8e93e9936eb3ecf3068de979c69dd11163283bb79f4a0741a4b4f7afe","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-decimal-price.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:b55d3d74343f25a72b96481acde2c52ae9a317a48f08aa7cd6dad5ce534f9e68","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-empty-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:54180e913095657a1b157b3329214499f5169507bc9d3e50d1214351c6dd5d1c","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-enum-format.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:e6127740d604c843df5f7e22d0af087b7122b466e3764b1a1284d05081514c12","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:6191d6e120f61998fa958c621f6fd8bd6f9fb94cd03726b3cf9f086bc37e88ea","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-nested-options.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:22f83694d35fd6942ee0075590ffc3c83d0d3b18ee26d719a310850211d86b8c","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:dd6c93720572190b7c30a4750359906d73673c9dcded21f4bde963ce733610b1","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:4c4ecb60e88fae6f6ecc96ef40e3c67a52bb7da8242f13971b36b96c38404312","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:a1606ebd04ea73e2ad7a9ac71042eb82f81463dd2664f72c253234751fe35b70","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-weather.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:f2c91773f18d94f0f9e4f0a663637dcdec315eba0f8ea0863b297c546a7315e1","evidence_path":"evidence/20260720T181418Z-d3446dd1/single-zero-and-negative.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:d06f7b651e4136d5d822fd2cf87000f6b61c125f36a380a150b6476748ee5c25","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:e568b2bc147e229c530cbee6a5629f2aa0f129499ff12167c5347ffe0616f3a8","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-boolean-argument.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:c16cc7482252af53686385cca3b75851d4a977b5c35940e0937d61820b5dbf74","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-empty-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:fded705ff4c7d5ccf0f0930e1d8223d5f692053aa88e8d54c3da8a5d532b710b","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-enum-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:b6cc907d659a82fca5bfc19489e31bee7c64223d390210310c9a38267b08e10b","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-nested-arguments.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:55fb2621351f69991f3a273072748e57901ff6da84fb6d83ef8a5359a476f9ec","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-parallel-three.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:fb22e9dcf94f1d5db3c53cd135a9a91b24c034f365839382061bba851acbfa0a","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-parallel-two.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:e09dd967144cb83e2e0f8caf977c6d5f9466831483454adc14d61ab6c2270f10","evidence_path":"evidence/20260720T181418Z-d3446dd1/streaming-weather.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:0bad89de5f8ec93160aabea68468543a664866695e4c37928c4dd3aaea1c7a2c","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-auto-call.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:cdfc94cc26f43bc16ea9264096aef47c6f335c1565fe31a8aeaf24eb6b4a228e","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:efa1fd70d3974a532d954a251fd459f93036d6d0bc96bf7dca869dcd5cfada9e","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-named.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:19066bb957cbc151f0622de3f6fd6aa54fa7aa41931d217ad79b862da02e44c0","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:39d5ac356318d03f4759545769027874fe3f039c363192f2d968cf26a9e4521c","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:acff52d9fd0e347593bb75504420a2189aabbd3de9cbaa46435bbcbc60fe6219","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-required.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:9b205c4c786c6f2669382d61d0decafdb72f6768714468642fe6736d2663bdb8","evidence_path":"evidence/20260720T181418Z-d3446dd1/tool-choice-required-empty.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":21,"timestamp":"2026-07-20T18:14:18Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_0","scheme":null},"revision":"sha256:4f6b83f30b62bc3d0cf9be09266db222805ee815c8fd7d8b38f863f655be78b7","sha256":"sha256:c8985d236593f7a17da2a3da49588aa951a9b1e57ce97753364fbf59e63af84a","source_id":"hermes3:8b","source_kind":"ollama"},"canonical_id":"NousResearch/Hermes-3-Llama-3.1-8B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"hermes3:8b","endpoint_id":"hermes3:8b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-hermes3-8b.json","run_id":"20260720T181336Z-c1bc4979","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:1b805164309386915fcfe314c727dd9b00fcc6f8670f01b4a78ccf1ba2065eb6","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:bdf93d6d196f5259b898ac30977de38c036611f88c0c073117d1834fb8871928","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:c4a325b2d232b40f22e177654341652397628a62bf9010a79eac7d223a225dff","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'check_inventory', got 'create_order'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:6018ca7fae2d469817f607413a417a719a26251aba48fbf9e8e5f641491960d5","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: missing expected tool call: 'get_temperature'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:6b3dc7e76e7724a1ceb522c93273d72016833f9b934bc4e662cfacb7fcc74403","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: missing required field 'latitude'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:3199cca20529f75322c1a76bb175d67d6f86e810e4b9e443bc95d8dc81035945","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected argument 'priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:73dbc605580ad9e3a17bfca547f0e1d9077803f2c51edf607039770cf9a4977a","evidence_path":"evidence/20260720T181336Z-c1bc4979/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:56465031a437ffa3e98af1f432fc7ad1fc9c4fa9228c63596fdcb143b62b85f4","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:9314e74cd507c48d74d81da02bd8ea8b17913d99a5ea284e76a4e928bb417b12","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:766735029a6f3db4e5e05ffc111d03f3ac886ba903273f860dad7e534986d3e4","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:68d968c4b127b5fe9d8cb4d1d8cc11e31d16f7b9d2d2dc662a65d9f3ef0b5336","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:ab80f77e78a9a0790e878a7b172ace2fa0ecd50f5a37a145dda019f749f2f6cd","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:91fc5d73a4775e993a68eaff59b00dc031df5d061661e5ae3d65025b02164229","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:b9b3918b9305ff99c7bf46d110d87c4055eaae0ca8faa280e9a3e8b961c89dc1","evidence_path":"evidence/20260720T181336Z-c1bc4979/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:930564adb0800f320315995ead36d23af00ec85d4c058f511832ed6308df86bb","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:8687b77c2d516645e9b72e9f56989b272869eb10095ca2ce106fe2682030faea","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:708410edf990ce1974790e74c359602c5297c9b419a4be2088f570c7484be425","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_weather'","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:2efde9d618b415cf73d0d6bee737d1e3f4f0d7b8f48a942c20bf5284d1941716","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:e68da588e56a462e145d9c7ed167f1d780e9c8a3d14284ba20bad32887894c1a","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:c82754f1c6a0ef0e84dbd3dd9c2c9d05d2ddb23574dc549cdbbce8e6ba766649","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:59b3c6be22bbe51b84b8891ba6e886dadc743b2c0f9c6c2a913322df86a5f0d5","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:6be7e2a12bf21f63179c236b983af4b7e8e2037eaa9ce3ed19ac5daa3b5a07f8","evidence_path":"evidence/20260720T181336Z-c1bc4979/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:ebcd56e4b435b63ab0b927db255d89047004b3ffdfea6d891715be1c1ae90dc2","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:84f1e660e4c13732e3f90fe53a6e4df4cec8a734f982b8ea5fc06744069be1cf","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-boolean-flag.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:122189647805561bccd8ff6832f9a02293fd6fffa39b8aa05c2124674474b5a2","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:ae5595a35b52e65f9195543d1211f537955fae56e72feedd42d91f05aeb1a22a","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:0862029275d92b98f043ab95acd84a4d4a3bf2d00b025a8ad940bc28d6276d90","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:1cf9fc6c24404da9f374ffc9c578d8f535cf4afaa3c31e2c8143b4e3a453cb76","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:b5e3edb19e7da7376c65146fc1bf27303d948a41b41873b31cb751deee113776","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:f0a2bcf0319ce8dd3838f73a96ae590b6c58353ccbdb9f6467a6b8840a6b7a40","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'options'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:6f161649f0622b65ae134462c40d28ea8ae3c7e394e5cdc6502bdaa19c2eaa7b","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:f500d6d2fbd89ea18f74a7663073a03afa30d2f4c445532abbd05402bef54008","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:458fd2286de8cefe657513e1fdf6b12f56f81f06504c5d86d8b2116584aa6b7b","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:1e0fba9be643a8710a713dfbb014d935f207a6801865b8786f9780ff9249ee93","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:a80d1a2736b17cf18824dfcb1065aa429eff57e4d8c7d642d5de90ca7fcebca1","evidence_path":"evidence/20260720T181336Z-c1bc4979/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:4879b6cb7bd0135d0c15a2f1e564e6e0e8aa902ad2dbd2be1770d3b0c1888d7c","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:3ead632497ce6efe2d91f1dd2b94776d1eceb65569ba113d2f6a5e410aa7c759","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:8f77f18f9b5ca0451a4b28af94d059baa9247462879fd1c8349fe6d5b65c229c","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:6f04881046b619e98a18a9d900aacf7ca006d20b33ba2db12dcfa9a638e85f7d","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:0658cd728358fd884be48b90a37067d95c6ddf6d08558dd82628db35fbb9d04b","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:8518f60ef943ef94368a02d3d50dbc6c6e78970880c2fc42b740a6e1c6dc0c91","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"missing expected tool call: 'get_time'","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:03b497a5b1baad2cc9e9de73889174ac2d52adb84ab236dbf38793c9adfd178e","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:493b9ca27ae275c8a79e6fea47c43d30e4fb126f6854e799bd59657960457cac","evidence_path":"evidence/20260720T181336Z-c1bc4979/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:6c0ff85ca2938d34956ec8ec7f432ea5d50f052328080051f2702e4650431176","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:aacd53d9187a2c6127ab8613348e9c02c829ed0acf9583f0b2a759591e9b7089","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:0e8fd1c3a8418ac7c39b01025beaa3963bd65d3c29b37c2d77bf617e1b338556","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:665a4fc055d822ad322b6d357e666aaa925571277076f4afd219548c4105b2cc","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:c90660db790e493acf73cfc5c292eea11060205ef1030f20a7cb4d36c2f502ad","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:97582e0ba5944af88ebe509acd0a2e102a1177af13d357a0666026a34248d8b6","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:a058c54f2879d3369ad64ec569d13d8d82ee2054c9850e4f474f31780223b763","evidence_path":"evidence/20260720T181336Z-c1bc4979/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":22,"timestamp":"2026-07-20T18:13:36Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_0","scheme":null},"revision":"sha256:36211dad2b153f09fe2b0c5f3e4e82ee83d27db68e3a00006cd934dfdc5057a4","sha256":"sha256:23e78d0ea1ab895bcbe63a12422a731cfd2b1a8bd124a1c91790b2b9c3fc9d14","source_id":"llama3-groq-tool-use:8b","source_kind":"ollama"},"canonical_id":"Groq/Llama-3-Groq-8B-Tool-Use","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"llama3-groq-tool-use:8b","endpoint_id":"llama3-groq-tool-use:8b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-llama3-groq-tool-use-8b.json","run_id":"20260720T181255Z-2c14754b","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:4f061353992d27c6f87a5309e86a2b6bdfc0582d9df8979d08fe6a6cdf6ad0fe","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:01d019cc76504292e92f27b4d1ca4598490bddf0034addf07664fc997b157813","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:831052ad49204a91750e6ec818549f34a652e82064fb8b9c999239cddf715bde","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:6d2307048e1554d3c2f8385697e5e0a54bd17d98216b596a66b2c76057ceb701","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:b715b1d10d0a39883f98f9455977abbc3b1e3a12b72e1623e14e4d2fc13b23dd","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:cae0ecb7485587aac4a9f0d6469d8f46c1afa632f66f428ebe7e7154c19a4e65","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:ef8631372e7ccce306f3e7b0e59db85dd73a95d69df02274c760ef400c3eda1f","evidence_path":"evidence/20260720T181255Z-2c14754b/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:2b525d89022f69713fa01dca01719b789a0d255e764ed38fa398b78ffb997fe1","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:884b373c10ba8838a14019578869636918c0f9266ec7cb1f4d08da81bec5da96","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:35e9565333366d3ce8662bce38a1d8216118c927ae72487ef492911bd13680af","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:ea16f05461ee58e36b07083975216828263ae3e31d928036874c94608e890b69","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:039da49de0693709b2829824f699edb6ed9ad33c4d1fd23dbb4c511a1b34f110","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:84527125f5ec42f1693b57258f0e26eed19f8dc10e15bb49acb4f62c0f036110","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:58aa78f1da6885dfde20dbf4d76c60d1f3b3ed16a2c9b05455f49d5b88228bdb","evidence_path":"evidence/20260720T181255Z-2c14754b/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:6446e422c6809916f35cfd47e6809bad4ba0ed8f8924e8077e35343caa472661","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:f2fc1ef19ec718335da5682c274723787424360323948e7b44a928f267dfec71","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:fa67e05a5dd3722be9003e4f3f51181477396158d8b91f9851936b239a66277b","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:c2f0f0b29f1e60fe25b3d00b6c4daa020feaf72cc4bfbb2ec3aa89cef98158b3","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:33c9147b3856e322bf0eebd9f289a58ce1fc7adaeba2014afdfa4e19d7a2f924","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:a002a42350a1a6a389ddd0f74db44b1bdc59c96856beb31f70ab806835d649f7","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:e01447bb9cdc7320811841174d5d7bea57b1429e946463622fef07ea43a684ef","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:96e785db58855834fed57fe6934b069d6c4ce9dbafbbdc1d28e574a9487b353b","evidence_path":"evidence/20260720T181255Z-2c14754b/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:aa36290266836f7c150abcca5a5780d3e2896fef1a1c074ef2047eac4791c50b","evidence_path":"evidence/20260720T181255Z-2c14754b/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:a9bc020949b62b31c3306e21be70400234ed4643dd36f5736436a6fd7afe6f2f","evidence_path":"evidence/20260720T181255Z-2c14754b/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:c388b1623df111137925c04df28306adec00ba74eb33387799ed49fae283d602","evidence_path":"evidence/20260720T181255Z-2c14754b/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:79237e3eca1269a03b8aa9a1bb6775e1e02dbb3409f990a826571bbafdb0fe75","evidence_path":"evidence/20260720T181255Z-2c14754b/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:5b4f2accceafc8b7eefe59b83d087d600007b00ac93178214ba1937e7f55845b","evidence_path":"evidence/20260720T181255Z-2c14754b/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:c042370d2a38fa71958e24b92e9b135adc280b1a77260c0c70ca200e81618ac7","evidence_path":"evidence/20260720T181255Z-2c14754b/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:c996aad70347c959311132919c6a94c1f928fbdba45ed35950d90f461238c1b2","evidence_path":"evidence/20260720T181255Z-2c14754b/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:3b6cc6e0b4783771f9d1bb21fe3f09db448a68232e8ead78ef7933221d22dba0","evidence_path":"evidence/20260720T181255Z-2c14754b/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:f425f79340d7025707dea00dfec741e0b2087d54c2fd1b2f7d2207a82e995a56","evidence_path":"evidence/20260720T181255Z-2c14754b/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:ac1e0e42954bf2cae23818ac791d8569dc0080f2cd15dbf67b5189549e50888f","evidence_path":"evidence/20260720T181255Z-2c14754b/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:344fc94cebc8994ba406be857af4690becc615c11bb43ff14facab54dc4acd75","evidence_path":"evidence/20260720T181255Z-2c14754b/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:182984e29e5a0dba8092c14f8b24edb2a75b9ce0ac174cafb14ce1a2928ea0aa","evidence_path":"evidence/20260720T181255Z-2c14754b/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:96d9338db4d2f399965c978d35454ca7674597545854a8e55106cbeee1fa1a99","evidence_path":"evidence/20260720T181255Z-2c14754b/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:c86206ac480e37c9a45e9ac7fad597698c02f2ea5b67f5d839ecea12f95d10ba","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:8a62027021ef1440a6b56c3cad402f9143d6f7621d8932f4d38b05381ac82fff","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:eda746f9cc56d363a753b3aaf20499b74a5007aafa33acbf65e3de8844e66f71","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:dbb3a40f2c8b0fcdd13fbba21687770dfc2232c62d7d166dac661f5ef93bea2b","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:12b26fb37ccbcec7ddb9101e941db35b13126d6dfa926bf5e39341fc74174b12","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:626b857105cb55caa57983a5db8e5788885c9fd8642c63ae3a99401909b9eedc","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:bb8c97d2f9963d364098a316b5d682ce3537a4badf8ec472993151ffd30941a5","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:3e2e76e38502c7c4530babf974a0871ff749f9115e79da1aebea4244b03d6f9e","evidence_path":"evidence/20260720T181255Z-2c14754b/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:015e20c53cdf224586d763f6c8d1c18e003a2425d7cc405fa444febaea8fcd89","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:38ee84ff7ca2fd99f53fc93bafd55eece184f1db8186d2403edcfc7222ddefb0","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:caa94c9ba3a981386ac0c6f53ddc364207cfa7de5f617aaa3bd9cf43f3a3ee98","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:7b6445b5e653370c2be85f38d619adbdbabef8a17a1130900b6ee241fd209749","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:c93bb382675491cb6a9bd5fcd2be9281f55ffa2b6c76f52da42ad488543711cb","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:17f8af5bc9ea13dd65a7118c2269bc95ea70b5e05431eb07f933fe40d31b6cdf","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:ca84df2fdc587e21038cef7611da918eedae416b7808054159d6286b23ed0b87","evidence_path":"evidence/20260720T181255Z-2c14754b/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":23,"timestamp":"2026-07-20T18:12:55Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:46e0c10c039e019119339687c3c1757cc81b9da49709a3b3924863ba87ca666e","sha256":"sha256:667b0c1932bc6ffc593ed1d03f895bf2dc8dc6df21db3042284a6f4416b06a29","source_id":"llama3.1:8b","source_kind":"ollama"},"canonical_id":"meta-llama/Meta-Llama-3.1-8B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"llama3.1:8b","endpoint_id":"llama3.1:8b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-llama3.1-8b.json","run_id":"20260720T180523Z-8f80361a","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:68bd7727dca0e501a14c49d8353738f459331c6bf756a5d84a838d223f2f8255","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:51fd9e52de9c81be51128b9671e6a1981d9e767b7ea7617151b171c52c3ef13b","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:cbe8a8836bfc415821bebef51b7bac860937417f952f118ed4aedec5560c78b3","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: expected integer for 'quantity', got string","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:e5e299f5ccaa7402efccf20b811a40c77c47849cf8a1f49778cd9ec4a38502b5","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: expected number for 'first_value', got string","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:309c8d7e9ebe5bbea31e2537a7531116c844ecbb0bcdd93214857b50f20751ca","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:81faa9cf8f53a0cfd4befcdd39d204731a8380401e1027be6e62b871bebf718b","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:d9af9de066cbe8482b1a3f48b5d5d69e695fea07d5c747d497b0308c1885a7d5","evidence_path":"evidence/20260720T180523Z-8f80361a/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:73e5a25c478298d38f71db96e782161ea4d97b5acea0db59ab46e0f5137195a0","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:e887236f40b49165ae6bc473970c152cb95e8720bf423707b1282ce816789f9d","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:e25a5d4d67afe01d9931e41cd0eb312a6c19b80cc67724beb7fd8c44cb910b38","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:bdb9d051653e1a90532c714f944babf7a94cd91ab5802c66d86d855d6cf1576c","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'set_temperature_unit'","id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:888f6aa910ce080ea70ecf020a4eb4bc8cbd46eaf376522ffcd8a1117936e768","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:c668a4f3e283b8a6f350f5bf512b2ed0ccb2967a3a10b68b461d5f896928565a","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'send_email'","id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:63299101538c2c68dada805671e395b95b26d83b25db618226fb1b15415c2e50","evidence_path":"evidence/20260720T180523Z-8f80361a/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:8169572a40721acc3a82e0674063d0131f5af26ddac741ae71b7748b8d57e3b1","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:a167891cbdd7d9b40d4bc1bab4064046dcf1e403ee53af5030699a8d6c58b302","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:13356d7a066a71cc6563ae8770719dbf2e7c375d09ab418400ad0f56b87ee19e","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:b66dbcd01b39839ba100e8b1816fe082b4ffd76a1e2d1061a52b436b218d57d3","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected number for 'amount', got string","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:788ea40d8c4e315126a07fe323bccc68cbd6533afb74d4b28f72a871873e4966","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:c563bbb34d45231f7ba35de877fbae97dea44875497ff5c16640f61314d7a53d","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:cca6fe09aa1c3607d458ef364d340d3ad8b13a4f9a597d99bdbe494ab00c24e1","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:06ea989d082dc605d39b928137b8add717682674188a8f84c1bff8d11bc814fb","evidence_path":"evidence/20260720T180523Z-8f80361a/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:1295d7df97086bd2172273953ac9a9c48b3000342bedaab0f434516a0a095ad0","evidence_path":"evidence/20260720T180523Z-8f80361a/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:c83d00c392da9d6f76f362ff5cedc1603e4a7f1faad9cfaa0870c8f3f778a669","evidence_path":"evidence/20260720T180523Z-8f80361a/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:e61f0a385c81bd3b0951ad290464cc05b63e73ff18d4a6ce15a4c487b8a53b8c","evidence_path":"evidence/20260720T180523Z-8f80361a/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:e3e0167c826a453d5c135533405aa82f3a2b7b3fb4d49559459d86f2aa814999","evidence_path":"evidence/20260720T180523Z-8f80361a/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:211d1184a64a69ea84e333d517b5e5c0144cfc2b10541a4310e7119c5e7826e3","evidence_path":"evidence/20260720T180523Z-8f80361a/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:fff2e49e928d778a993f860c50f58810656f317702e98a89a655b57c6fc64bed","evidence_path":"evidence/20260720T180523Z-8f80361a/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:e0b463c4c6568c2920f5a9838564bfc2d54298ed5a7970beab54d94736480032","evidence_path":"evidence/20260720T180523Z-8f80361a/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:319d78100279912303192fe9f4f958ad575b55e5eca3cd5058c7dc67238bedb8","evidence_path":"evidence/20260720T180523Z-8f80361a/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:0b2df06f835aadf1d289b1b603493dbcfda2cf11fd0b77858312989aed738f07","evidence_path":"evidence/20260720T180523Z-8f80361a/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected integer for 'limit', got null","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:ca3e3e5a8ead5fdb811892e0487836dba7bd4fce67b8366454dd534b00ef1614","evidence_path":"evidence/20260720T180523Z-8f80361a/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:48f2fef7904815344326363bf1274fa499b1892f4b8945e0d36edc2bcdb97d1c","evidence_path":"evidence/20260720T180523Z-8f80361a/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:aeedf1a6a9b252578cd7e586ac66ddf0cae586ea94773ffcb1883be8396d555e","evidence_path":"evidence/20260720T180523Z-8f80361a/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:0d0c06f88df41773f1c704d74a97d2cd7a03c3c0b88eb60406bf64b5b5edd320","evidence_path":"evidence/20260720T180523Z-8f80361a/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"schema violation: expected number for 'maximum', got string","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:1fed5bcc7caaf8dc97bc28994e8404a9f76c7436fc47255e94b1a893bc098ad9","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:06cae10c00ae72dfe7ddb0fb7a81e27eb997caa07ce73bd8371edb7003548571","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:92573f7bdb9342e0ca2f9516df04280b9d3cbc25143fcbfde39304d84aa1fa1c","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:f38ee18ee8cf378c869c0a3eb077e63f560e89d781bfe30b994510f682749beb","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:6f8314abb7bb65d9d76e730e434a236f827478215ad5e3f53b084931d2725006","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:3d6e16f8cee99d3a7d32f94f19106212066301c6e6df09cad8fcf00650b4df92","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:0d7469bf2d1ae5a89b30bc9347644c7a34eb927fdb76e8cbc7b804004c76cad2","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:a13fd39d6ab1ffb535de9091856bbe7b7aecfd34bf58d354fe44a10947a23423","evidence_path":"evidence/20260720T180523Z-8f80361a/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:69e9de24b7af92b2b6ab9f26f72b2a3257439e22fb96c14a2ee7bce426bd8996","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:c26f61ae2ee0cb75bf525e9f4cadf959906d243c41ed940cd3e2f374c91fa274","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_temperature'","id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:2d4eb32e4e4887ac1f2e8c7f6489d53ce6d6ed3a6ce9c45462bc513122ec1507","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:99ed682aec338658a47296dc5e161539d4d30390fce6295582ea3ba0c4e65c0e","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:4a80a3891d3652564fa1392ade364c9d471b2b10283682d7d44fe3110bb05c1c","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_weather'","id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:f38c98a1afd1a581f92a28cc84dc008365f0deacaff002bbb74764bc40ef3fb3","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:9011490bb147d4c2d459dbdd0d232383d9ca2f6ee171eec376e6e25453129e0b","evidence_path":"evidence/20260720T180523Z-8f80361a/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":24,"timestamp":"2026-07-20T18:05:23Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:6577803aa9a036369e481d648a2baebb381ebc6e897f2bb9a766a2aa7bfbc1cf","sha256":"sha256:f5074b1221da0f5a2910d33b642efa5b9eb58cfdddca1c79e16d7ad28aa2b31f","source_id":"mistral:7b","source_kind":"ollama"},"canonical_id":"mistralai/Mistral-7B-Instruct-v0.3","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"mistral:7b","endpoint_id":"mistral:7b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-mistral-7b.json","run_id":"20260720T181455Z-e6e6d1e9","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:1096f9080f645f37994258bfb6b1b9a3c6c8581d7d3c28a6a9d38d7a5a837bca","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:1152128f91e9fe0753e88de2a82cf4cb9a8ff8b9febaccdd4c479d7df98fad44","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:386e63884693aacbd12fbba690cba8d67f31e098dfa0c740cc2daa138f73ece4","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: missing required field 'quantity'","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:b729f190a9de1acb85e3e818528eb2aa19c06d28e919334053a3eb3f5ac47981","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: schema violation: expected number for 'first_value', got null","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:26a69fc40f26f356a070fad5ec4aa9ad68a65b0660fb59a1f59e01109e69a7ca","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:07df181f757ff3a45aeb51f5b4d9eddb80b84ab1aae0a500ef2b2b6c3d5e258e","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:84c9997c20825bec9c3456caba058c741e27b1ad20298a22008b9f3f6b08d7ef","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:3f51890b612b0396a84c90e16d77f85816031a4edea967c7b3a3f1f2b90defd2","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:02230524d934310fabf57d404cd7167c1408b8a48ca52023051cd51d122b0110","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:7ee884a72ab8705f9fd2b799eca8dd1e702fdfdf886ec29d5030853f96ccab8a","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":"unexpected tool call emitted: 'get_weather'","id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:d4161226bf087f961c513fc755d140c4d6be8b232a64ee75c41e4650c9cc6171","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:1ef6de3c3934e6e3326490ed43c52935ee3e05352280a1e2f07918fd161a8ee1","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:e7f4bae3bcadc9f5bc3d1ebc5b43b143be993e018431d449fa302b05a108bba1","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:0f390599312bef3964c179f7a05e8a7f3a01e49920d02fc5f964eb8e074493d2","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:0f32653c69c0f8c1ed74554e219e6cc568349e3d5fb9e7cc4b6417abe95237fc","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:744a34b3f43996b1005f48f37be7b67e64fd5f12e2d6636082f4fd5018a11d05","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:87151caf25edec1256345fba544bb438d036c077ef7ea68c710cec6669ad19e6","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:ee85086fed6f8910dc0c76f3a4203368217989e6ed50e6d67b3f25e55bdafb12","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:bb278b487815aa7698ceeee73543b25c478e0e0641eb8cf198cca18271e0be5f","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:0a5a3cc206f9634fde52204845c5a170dec91f516e88b61a44960d250b006ae3","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:3c2724ae83782debf84168e6f8348d3e77a8c206fb4202e505b4102cfb85938a","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:fec676b22758006137a20ee17b17c24046374126e916fc6e8fabd6ae29a22641","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:f99d33eeebb2b766b6e95eb5a2479bbf1104eacaa1e39c97c6452a2e96ddfe16","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:9f916c6c4d28a25926c44b6b1da5c6ba70a973fe260c042df8bc8e755ac4183c","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:3cdde8af199690ec3379d80d4a1b0facbcae70619ce3efa3f51731fe9eb33e00","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:4c3217e1721bb2765ed387b66bf942eb2d04015b10aaf09ac521edc84b4b880a","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:38a8fc69d02f520e04433ab010f5d1a8fb0abec4ddbeb5c4221275b97515f5ff","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-empty-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:1b73d37c68a22f965d3a2f7da8e860cff346f7cfb0cfa0b03cb8f90b72bdc605","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:422900f6149d476ee21138a63ba6847f3da205f0e4d2a373e6a33191a7f40659","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:570065322c1dd4d5f93367eb2f35aee17ccfbbca95a1286d0b2affa410a03b0c","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-nested-options.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:61a0f6ac913cacc282ac9715ae6ace14f078bc04c1aee92b57ca5e14a51d0dcc","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:cdfb84d44d3d2ecacd918b11390020537c47e59f69b38f6f0d7a88014c91ca91","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:bf89bbb7b858603955e49605e7053f45fdba4dca34591b2c2c32f818078cefd3","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:56f935601251300e88820786333cb7d816edefdca881e49693b8260f9b8e59e8","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:cb19372d9f7f32fb2c10ade5b816e04db37e5e0697af09c38199dd3f0e4404cb","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:f3f8fb0674e53e576803fd182d92894556ed78b8dde90c0ea9f2f3af5c209967","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:546e867de70e0d9dfa28239ec1394c5dc5e3a405842a0316386018565a0758b4","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:f75b7d959f4ef4e030a7775649c5dfa30b53e19ca91c72ba2f4ccbd6309fcd93","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:89b286e59e468ab74580ad4aa1ec78e439072b5143fae7ecf15f7154c6057be8","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:2a866bbafd40403732dd7c62a167624663fef5a8b8edddaa257ac49f6fe79c55","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-nested-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:6b06956c6f3851785d101d28fe2bf0ec79c91ead0659dc6b6a20210709f5768d","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:cf8bf08c1eb3b9f256bbe05b2d1bad05f3fc7d13eb585d495ad97a0768683c1a","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:8537be346eae349c323ffac7e16aef88052f6008f681fa2fc68ca1bc92bd0295","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:049baf37499e10ea4f080b738a85abd3acba6dee2e419c6e57c6d8d9c9ef506b","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:9a8b6d7569269ac8c6f053d6d290c81916a6aa266dfd404384af69e1b6357c94","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:0cf9e5446747c27666486f0a95faacec7e56ff1627562711494a6eff3d11c6ea","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:8b2631eb9822fecbd0fa4def7b3a1fff238f787604e6cbce1ea7b5a83778af34","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:6c57815006c9c295313060ffda393f6fe3aaeed9542bae80801a8cc0922f425f","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:872e23f3ba97a90cef247236a61422985bc1fc836c8b188d77647def71ed5ad1","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:01e35d8c7f99697475e79460c2a5fbaf1ee3cb7589473b158a203893b30a5885","evidence_path":"evidence/20260720T181455Z-e6e6d1e9/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":25,"timestamp":"2026-07-20T18:14:55Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:78fad5d182a7c33065e153a5f8ba210754207ba9d91973f57dffa7f487363753","sha256":"sha256:3c168af1dea0a414299c7d9077e100ac763370e5a98b3c53801a958a47f0a5db","source_id":"phi4-mini:latest","source_kind":"ollama"},"canonical_id":"microsoft/Phi-4-mini-instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"phi4-mini:latest","endpoint_id":"phi4-mini:latest","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-phi4-mini.json","run_id":"20260720T192441Z-7f3fc9b9","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:da027adfcc4bb3b55da5305176c33b5b653235b1184f770ee8ac43704cba9cec","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:4a48d306c3895d73cabf4ba5f9059137e9ef97b98ffd5066b60c43464d36aeff","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:7b6e0fb7b15e055456320f1d0b72c8148b44c44125410c24488835cad0b94bba","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:d44d7c68d1fbace6ed90d9634e5985bb692938a8584956484a51ee49a4256f4c","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:3fc7369c904e9139550bc3f130046cd3a11b3c3afa56eeb62d2e4119b26b3f24","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:b757a4ae8d2bcfc8034560fc9b4f5249c94fdb9ab76f3bc0d5055d1b204484c4","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:bde70cc4701cebc7a59bd0977489d6566cd8fa761e29470f6ab788ba6a0bc1a0","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: no tool call emitted","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:061b816c39cd1f564ef239b6419c97cb95e257f218067e9ace12e4e98ecc6aa0","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:b1a1047aa0a75e3292005c035ec7cecdf7d7967745064df2273b828f16f3a9eb","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:d1166f9a691e25bdc7138a0680f5bfd9256a7292438a9882e5f45875da0ca8e2","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:27af97607a27ed86601ebe54ddc17e621497e221e14f4bb2a2484133eb0ed0ed","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:d5675d287f4f76f5e357f92ee3c63c7ab7cd9fdb8bb35d762ed94f8df7e84d32","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:c3965554df5cde67931858add1d89631762903c64ddc8e12974d52137594e70f","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:b95f97fa46d8f66eca2979c3e6d2703a2e82d7d76fd6ab664340ee92aed783ca","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:97e19833f4d31e7cf853349664a9b750570bf102dfb07d46e4a6be432c17175e","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:8f5424a0a73f2121fbbc7eba4c5315e08922fd68842fe8b769ffe3bfd3dcf8cb","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:e037b3b5b07a612ef00d5514586857342d3b0bdf45487a4243c578084339d5e6","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:52420b0c74416a0a6bb48af4e5bfff32c97f6a176fead71f5f450f8556a40db6","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:bc246d5134bc08cf79af4daced90ee5d2fd287a049138b262021bf874661c102","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:1e62e83f66011a40052610789b4b5fc01234a3fa58edcd171c119df90bcf2eab","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:3dca1bb972ddc18d3ba487172750ca707b7053061e69d2aa5e683689e70e27fc","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:e2b2c61eaaaacb7a7b93bdaea93b8f8a4b51ead6bf8795becdaebd4532323d0a","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:b2380845c721748e8c9da01b5379a0f76a6c84635385e78e5b9a329d2d37ce50","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:2b0bf1ef36ef827795a080a0b56da715b90d5093802db45d05e6605639805478","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:19761a5212b2bd0187859f4d111233f7e8a62b839c77868e9020375a3b05a67e","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-date-range.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:1dd55e399bf1208138e0c5617ba38c60e139fc0f75ea21b1cb45ec6a9dddbcdf","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:b61e37e74ff45dd93191184ff94cf712d668906a861fc4bd56299a01936954b6","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:e3f06398a4ebc3aa1d609d25a5c617c02c075d89dcfeb412fa0d1d954920e168","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:4e1c55314bf92ce4b9612a8dc3db7791464de64b9db1d42c4b58bad3b0fb0c20","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:1326df3dbd16f1ba5e82a0888f09ba7f26574186390104117321d0340e2861e0","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:7443ee02f29170c1587b1262abccf307944bf5e452caaad56a9e435291532ece","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:f505632fb5f378c04d1faa22e87aa8fb1913cf31090ccd60c228f75ad4ad785a","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:4672484de529670cbe501eab407c8982c1d578b4e7b2260efc08d8813b334e73","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-string-list.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:f9f9ef7da3256fe157a1fe40bce9db8ef3903edaae5bbbd3f713e022e7064cc5","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-weather.json","failure":null,"failure_class":"unparsed_tool_call","failure_reason":"no tool call emitted","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:42148e1dc5bafd048432ca5c3a25760afc1f13ad6e85b1c3cbd46de9e143e6f0","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:dcef1d03bf056acdad0774a3de0ddfe101efaca2c21eb029a9239221e84381d9","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:e84a137a2a8feb998cef410f610b6e2b8697c7835eba6d34be616ab0fd0d8280","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:20b71367a80ffa0dcd888244e1c39a634111e82bbbd1d3d481f3d5e9a5e4b936","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:0dec768c51311071dbda61a73cfb8dff064edff4048a3b08f705474d3b2e4a38","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:64b29f14a16888168851b2c39f62e63aa384fb630f6300ef036261436fe91c58","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:2a0eff56be51542db050e6be797ef67592f600f3c495acfaa4642fdf3766886e","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:f31259f7d5f739677f04050457df8ade1dff51ad765a3bac712418a22a2054cd","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:128ac0be67157e310a2593d8db0ef2adc3e28e60bd8bc497c6f06e76ed8d3cfd","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:f00763ac9ea5280f48bf50fc0b40792af7743b50fa7362dc9794a744901e87fe","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:1aa59c20b7225a76f853b51b9963d0dbcf5796a3ac77ffd56a008a30ea5b3b05","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:5320bc025dafda42fb0d4253aceaafd4278fb3ad028082918e465edbb0b3ead5","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:4edf3de69dfd389416c4a99638d92a6e1572a5dfaec2f530d02407a6117d2ed2","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:2a9fdcf9ef0868db9f2388824d203d041124bb94f27df24d0b38658f4ae32f5b","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:b0df0aa66c262421940beaa3fb4f4ee13ff53aec195db56a4d4e2e3841230cdd","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"fail"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:7572f36e0ea8b465903f0215283da715537fb4896a4f186107c4045dab154a64","evidence_path":"evidence/20260720T192441Z-7f3fc9b9/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":26,"timestamp":"2026-07-20T19:24:41Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:845dbda0ea48ed749caafd9e6037047aa19acfcfd82e704d7ca97d631a0b697e","sha256":"sha256:2bada8a7450677000f678be90653b85d364de7db25eb5ea54136ada5f3933730","source_id":"qwen2.5:7b-instruct","source_kind":"ollama"},"canonical_id":"Qwen/Qwen2.5-7B-Instruct","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen2.5:7b-instruct","endpoint_id":"qwen2.5:7b-instruct","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen2.5-7b-instruct.json","run_id":"20260720T181212Z-f860526f","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:918483029924acaae32a7cf8580f7494c3f5f8d6802bddea0d6dc58e7aa39c51","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:c33397a4322a85e57b8e7315ef4969ae3285cc92a1090c158e2c822bd58de327","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:8a3b8a361f5953c659800066e6b76e9b44596c7873d987de6ee498551e849a31","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:07cfe59f8f8cd9d4c8b9ef248e3ce32e5cd9a2229d6b51418683f1c3d956bccb","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:2ac53f5d8d0c89162e5adf9ec47270cedacfde4a0bf07468dd687023cd53086f","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:85b9251f58ac7f353fc4d77f5c471b1aa0556df4061f21d450586ee7385eae1d","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:6f25120a4009be5c40e494930e98e630a6009621d45121215ff5b22e102b54d9","evidence_path":"evidence/20260720T181212Z-f860526f/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:7c8d4a9f0e196f3dd1e0cfc79024971b2c1b4771d2b41de3e1888475125aa6d9","evidence_path":"evidence/20260720T181212Z-f860526f/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:c6cfb915515537fdf295f98a45f60f4e8e0e11ce4dca90c19e632ac5a9412897","evidence_path":"evidence/20260720T181212Z-f860526f/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:f5cf30a142a3fa4f7b7fa748e763cac2fee7bbf0c9e57d6ad4b8a467ec3d3b91","evidence_path":"evidence/20260720T181212Z-f860526f/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:6d8541e40f275e6a7575c5b91e6fa0982f95f4005528c6ef82bffd1e9b5964f1","evidence_path":"evidence/20260720T181212Z-f860526f/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:195f2be32ecd92c183b90ecf850010f1544c6a946ec1fd97679f39f0c1be2004","evidence_path":"evidence/20260720T181212Z-f860526f/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwx0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:5c8dbd5b39c4c0cf405d2850abaa54916d2426e44011c2a22940f16876242eaf","evidence_path":"evidence/20260720T181212Z-f860526f/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:a88f4f09c4dd5c979e20973716c4000704a074031782d0412ce5771c199db37b","evidence_path":"evidence/20260720T181212Z-f860526f/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:cb2bd13eea64d287a3ddf5d72f1d58fe6200cf283d8c7a3cec19f263bcaa7101","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:e8ec388c24f18cf57820880590100becb49852c6171a6feac3bc2f5ab4829258","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:5726d0d5b83bddd7ebecdf9fddaffd4794140a346e4a0a17ca0cdb55e7972ac9","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:ec021f93278e8552ffd478bf3bd643835d9fb76edcce7469152cb6f9e5336c1f","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:5bc89ed14a2977a69edb0418c6a6e1a7a346ed8421b324cf5574762458ce15a8","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:af9ffd3322ae51f0c7afba6710a06128f2ffc98ba5d8a871b52895f401a9d16c","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:c20b6867f78fb11d26c2cafe1e8344ef483e1bb2d309a29d132a47e808e45da2","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:a61f4b65cd841a523de8cecbaec3e4a7068f304ec04a2f778bae2cc1a6d5355b","evidence_path":"evidence/20260720T181212Z-f860526f/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:e975c057011f9513e07d41a530bc99baa00e087bbecbfa1fa966d526834d5071","evidence_path":"evidence/20260720T181212Z-f860526f/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:0817a9622061d9ca05ed9f143b00e962eba9a407532387d1923d9e84b75e8d15","evidence_path":"evidence/20260720T181212Z-f860526f/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:6361dac534d03ee70684f7afadd351b9717de4d1f718ecc807d7ec7a185e1701","evidence_path":"evidence/20260720T181212Z-f860526f/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:d6e2fe2941f1f3d8efafe7613b77fd8b0e9d28f864a420bb74befca1f9ddb756","evidence_path":"evidence/20260720T181212Z-f860526f/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:ab07edf1cb0c8fa9cc78ac755c1dc2cd5575855cd4c2f6d453db7e10e7933a3b","evidence_path":"evidence/20260720T181212Z-f860526f/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:fca7b802c8a32abeef984ed2ccced662136f633c6cab86a9f5ca696198c03f06","evidence_path":"evidence/20260720T181212Z-f860526f/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:9dd95dfca7b7e4247be93eb232bcccb4cb457447098dcb0481c5d4bae3f319ec","evidence_path":"evidence/20260720T181212Z-f860526f/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:9cb828e02fa2b42c4f57e28e23a8b17e083fcc9302c1f52b60079cffbdce819d","evidence_path":"evidence/20260720T181212Z-f860526f/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'options'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:5ee27489eca17d65f119193aae8456120f0e2e276699f3abe19fb5bb18fceecd","evidence_path":"evidence/20260720T181212Z-f860526f/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:0388637ede91b86cd3239cef05a0edc9166e4565f01cb81441e19d25b8c9cccc","evidence_path":"evidence/20260720T181212Z-f860526f/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:8e8e9742643e85c02cdb7c16ead20a55dc56451d0e92e2490e01c91a92091bca","evidence_path":"evidence/20260720T181212Z-f860526f/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:5189d7438f0e7dac0e617e74b208910f5e61c59f7659c1cda8ab78145fb6c98c","evidence_path":"evidence/20260720T181212Z-f860526f/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:3d2b24b2cd5335bfbd33c9fb8a759f2938eed9816cbb97896aae82d6152335c5","evidence_path":"evidence/20260720T181212Z-f860526f/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:f14fda2c749f0995f9c5d918faa0e284e3cd3be3af46c7db2ba78d3088138afb","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:91cd570e284fcc23607fcdd327da1edbe10bbed30134a48a168cf752950345da","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:0222d3564c8a636fa695d6f231114d2220460416274db24d35aed8327620c5fb","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:5ab09d73f5d519460b12ae86647f35f3917af8db8ba058f8c2251f2d89f22472","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:6bf56429bd534221b1e26607be4d3a935650da3c0d4a8df1bf64c715f587915a","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:32e12547f8454223db84e62701ad648b3a827f2dae10254af9555210a8c971f4","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:8183a8b67aba0280fb846f80fd9e91a045613d0bb4528275357a27af4f2fa9b9","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:02845bf4f8cc9a056b2f646cc899af76e8536b6e6a648ec4b8446dc5e6ed5537","evidence_path":"evidence/20260720T181212Z-f860526f/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:5923a49b59bd6b8aa9f3c8886547427ade4b9ef4f046181326d9c6ad4d0f7da9","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:ddad5f5acc0dc53eaa6b470b90e2f910ac94f913e895fdf8eaace00b5539ca9e","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:cce1b2e15993be5734fe7c44869a2ac3e2d78afdbacbdcaee005d2ce14e45fd4","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:0e1fd2255d6d0e44e91fb33ad772c7e38e1ab71942f1fdf0d73d13b53554f485","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:44bd57664604a44e3e823862a2738fdc2b4e96a63af6d1a2f03e8c45374e0d18","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:2df891c3559daa4a879223ea4022c16ac036d91b7eaffafeba59583e0134329b","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:45841928a127a9e58918ddeab511bf27477877cfa9ab302d946fcb6ad0563c26","evidence_path":"evidence/20260720T181212Z-f860526f/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":27,"timestamp":"2026-07-20T18:12:12Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:7df6b6e09427a769808717c0a93cadc4ae99ed4eb8bf5ca557c90846becea435","sha256":"sha256:7f4030143c1c477224c5434f8272c662a8b042079a0a584f0a27a1684fe2e1fa","source_id":"qwen3:0.6b","source_kind":"ollama"},"canonical_id":"Qwen/Qwen3-0.6B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen3:0.6b","endpoint_id":"qwen3:0.6b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen3-0.6b.json","run_id":"20260720T175757Z-4b64779b","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:c94bed46045b634639896664ef3f362512c01e5bce4c98ddd3f3240372799bed","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-calendar-followup.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:8c558640ecb9c406957e09120bd067785efeee3f9abef5ef0e46b66b640e9061","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:f646016f1eb529a2f0307c8b2076cb1c54a3ecedf8c632ccf7d6bbbb9ca9644a","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-inventory-order.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:73ac63b3dd00f84bf62b2a9bab53ca64fe60d50b6461719cc5c090c57214f41f","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'compare_temperatures'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:91fa0bd25025df65f832f993e6d6cf79f3f121a27a000497763ff8a571bcd6a4","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":"turn 1: wrong tool call: expected 'geocode', got 'get_route'","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:7e7e86193a50d3091b3fbe964b3597f1afab5bb34cc984b0891c85cac4dbf508","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:75b4a0493f1424c2caab5c9932c4fb35671a2b6c38c24290f3c84621f35b1d46","evidence_path":"evidence/20260720T175757Z-4b64779b/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'find_account'","id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:d0bbfd0f7492344ed918edce03b733919188e46b36296f6d9bf829d1ac1d41cc","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:e7a0da05068e01b1133260e3f4ee77114b37a63ba7a57d7a42284b5a76afb708","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:555b504e8e2573e12f1dae4efed6a152f7a1a9b0ced1d363cfd6d4850e647a9c","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:f562cb1847cb95a7b1061bf0b8564f38bd02f615e27623983741b105c7ea7a88","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:12d66925f8c7cfaa817af1fd3de0da612f59d7a896d5ba84fd129619332296ca","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'token': expected abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ, got abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ-abcdefghijklmnopqrstuvwxyz0123456789-ABCDEFGHIJKLMNOPQRSTUVWXYZ","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:47a2578789adf5ca744c3e26918b4423532272392b8758a77e731c576c6d3cfc","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:c7911001cef24ee439bb82e0fff1440fa6a1546fbd5e07ff8bf7d195108a9ead","evidence_path":"evidence/20260720T175757Z-4b64779b/negative-unicode-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:c47822f7ebe3532b9e3304a2d0e177074860f08df7329b8bb503051f68bfd1ae","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:26cae1d342438971c48d86b3f938f3d775900b4df6ac06849e4ea357c08fbf8c","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-different-tools.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:cea9f98ed6611cd4dc8605093e0039e2f94bdce6a23e406ceef54090e0a4d464","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-three-cities.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:4c9030c81a705aa69d1edc52baaaedfc4dc2bd6410b7cb0e91f071552fec52a0","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:c2674a93301fe88c0a46fa2abde0a7366f82a4b9f4ebcb017ae2838917e1dd87","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:4c513a20826294e55fc20cf4b748c0ff748668d9e3908a26c817fe6a4424deb4","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'room'","id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:dc3409e476aa323a2d4896deea6ae04f73bb90b61a0c5862469c0b996a3730bd","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:be57ab11fecf25bdacc0a8e44dbc3948d0b5ec18d5153905fad86b82a53e0c6b","evidence_path":"evidence/20260720T175757Z-4b64779b/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:eccf641111470453f9829eda1bff085606a9eb1923e0d87713d2e3f3da2ebaf4","evidence_path":"evidence/20260720T175757Z-4b64779b/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:01dcc66b65594dffb0ccd85ed898d727c260fe72d181a03fbbedc540c2a7b86d","evidence_path":"evidence/20260720T175757Z-4b64779b/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:80c711559c361173e99938b382e8833379390ee721b03c731150a4387ac7c82f","evidence_path":"evidence/20260720T175757Z-4b64779b/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:d9f719a69fa283e0bdfa9e1f3a4d5ba043104747e70e4ab1ef82b6fa6ed9b0d7","evidence_path":"evidence/20260720T175757Z-4b64779b/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:47c9d965845124f37d8870d6b77542d7270ad16f19cf13189c3a1de672060c19","evidence_path":"evidence/20260720T175757Z-4b64779b/single-empty-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:6e95c0e6776152ae63df04a771c3e40e83d18c85ef3199616ec17c7050298815","evidence_path":"evidence/20260720T175757Z-4b64779b/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:353296629668e91bb7fa3dd13c9346dff4fe2b3f568d2638fe5aa560c44e49f5","evidence_path":"evidence/20260720T175757Z-4b64779b/single-integer-limit.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:5b7324661c9c6503557b27f0d98456317ea2c20d44ca89a53e767276a437513c","evidence_path":"evidence/20260720T175757Z-4b64779b/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'prompt'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:73deeb7cf205e9ef4e59345a7be53897a534c0f2f72eb7ee8c8c6d488f0f9b16","evidence_path":"evidence/20260720T175757Z-4b64779b/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:7922d3a6df8d536db6418fc16f998623d01b9b1c0b99cf2edcc03fe2166395b3","evidence_path":"evidence/20260720T175757Z-4b64779b/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:4cc7c8b1c9976f944e371ce7f9a6ea0b531e292b4f97fa1f176074bed4f6cc02","evidence_path":"evidence/20260720T175757Z-4b64779b/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:a1cc3589999e01c2b41d4b94307aaa797bb695d0aa240cfe8da56e87bb9c3767","evidence_path":"evidence/20260720T175757Z-4b64779b/single-weather.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:6a17a18e0564831087fc6a0181c8af505495ee0c2f9d4c7aa2a38dc2349c927a","evidence_path":"evidence/20260720T175757Z-4b64779b/single-zero-and-negative.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:d29d319e56b2677a9598627021accff253fdbdce2d4c4fd4039211ad4cef851b","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":"unexpected extra tool call: 'fetch_records'","id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:98a0fe4406b181a535562a2361a6004aa7de2e454d49abb4507f89c78952da28","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:2e7fd785a879fbe62cb3fa6b13966cc3b2083def5edb738bfa8634faccebf7d8","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":"no tool call emitted","id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:08a33f572a0817d48bb29be7860ce27b290726436a92fbfd05860e6a94fa21ad","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-enum-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:ad8a509310a96b08856b70b3998d38696cb9fb31eb350a60e5bca63cc26d27c4","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:e3fb950e924ef124ac1d3fe0a3a5f9dc92f1392c98990d93b1d4f3b45afb0f49","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:a898f7cf9f8d8808c26eb659269271454ff6fdaf2c7ae97e72fcdae9e74b70a6","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-parallel-two.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:1248a24d68205140fd199feb272d6602f535a76f0e2364b4ea848cce8f7f84d7","evidence_path":"evidence/20260720T175757Z-4b64779b/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:4379889a6bcb31523246e52309e779c795f18ff9451fb3bb1e45d086ec500e5d","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:1b68ebd08a87a94fd6e838ab2286d6a5b06d5e69d8b56b343bdd7cb47914a6a6","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:2b683daf556deb2725196748f875891240aa0d569ecc448b563f19d47bd6e849","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:ecbcc813efac7f647d4be924a4fa4097633afc4ab56ce1a22a794925a31105df","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:0f6cfc621874fd55ec17e4293f7a9fac1e00c3883617af03d4fe987bd58c5c35","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:88ebbbc6b90472b0947f3fcd4a5d626a200fdb774a0f67d7d9e0750e74fcc6a6","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:c6d1bc64567a424a949b0dcc65cf57f93fe06f91b7687dca501b428ddf68f3ae","evidence_path":"evidence/20260720T175757Z-4b64779b/tool-choice-required-empty.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":28,"timestamp":"2026-07-20T17:57:57Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:8f68893c685c3ddff2aa3fffce2aa60a30bb2da65ca488b61fff134a4d1730e7","sha256":"sha256:3d0b790534fe4b79525fc3692950408dca41171676ed7e21db57af5c65ef6ab6","source_id":"qwen3:1.7b","source_kind":"ollama"},"canonical_id":"Qwen/Qwen3-1.7B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen3:1.7b","endpoint_id":"qwen3:1.7b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen3-1.7b.json","run_id":"20260720T175837Z-50770d44","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:4b52f8fc6b7825ddc11b28de4e594784c5db2940c4dfce0349e77e3a4d03e2af","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'create_event'","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:37a2255b9205f85661d3b89d420ab1ce2e1141fee3b75bb1a0bd1582418e699e","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:5d04d2ce823ec1586574e80e80559209b7855ddf9c1b42d67829e2d552af9d95","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":"turn 2: no tool call emitted","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:e37fb67850810b527bca6730f835a0147ab5ad96760941b9687be1340060a914","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-parallel-followup.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:94cc9eca20e7cf954bfcae261a756196d146919df06bdb92fdfca42691a2994a","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:b74a48321b643a05835235bb507b12e9d405316f7e88eb1e8ec5da20edfe7339","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:dbd0ec6799e368e9b35746c3a08d06d0a1baffe6f625cdfc660d284f502d5c3d","evidence_path":"evidence/20260720T175837Z-50770d44/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:2d9af4f31da8a5cf32077df50694dea9d8998ccc58dc5813b4303366e0b75233","evidence_path":"evidence/20260720T175837Z-50770d44/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:8cc50983d8bbfe215bdbfefc5e07b04d32b030059e7dd729f52c98bc8e065ec0","evidence_path":"evidence/20260720T175837Z-50770d44/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:bef03ba6ed40af6179bd708bbc29daea0abf7446d571b84208ca58a323f4bbe6","evidence_path":"evidence/20260720T175837Z-50770d44/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:b8d146f3d5398ff78c69a244383d7f69983cafca8f821256d294176de40e8b26","evidence_path":"evidence/20260720T175837Z-50770d44/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:252b1c80035986deaeeec8eea8a5bd8a539697535b024119ffe392ea6706e230","evidence_path":"evidence/20260720T175837Z-50770d44/negative-long-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"fail"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:706258de1f3bf09f007458894d0c1a347eb280370a8411c3058cceaa47a380e5","evidence_path":"evidence/20260720T175837Z-50770d44/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:7feeea8316853d159910ed773bccdc3e367376a15da359382b4cca7343d255ec","evidence_path":"evidence/20260720T175837Z-50770d44/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:11f5a4f20ba99304ce49f58083cfb5a3d4b049a555f31a286f72099be17071c3","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:634a7de3f9b8ddd8e3c82e115266534e0ac3977027b8563d9085d2281ba6a67a","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:40dfd80d183f22871a0e2ed5ebd537e9e9a5c16e4a7766b17212675a46be4c86","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:04f26a000b47dfcef32997c439cd27916a1e88681df41d4c47c0797879201cc8","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:d6f849538b20f77554f7915fa7e585d3f46056d21c382a65a602a1c451acda5a","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:dcdba3b8506a04f6deea22b22481715d908c9d29ea3a206bd73109dea4fb160e","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:5548cae9fe68a563063d7eb1d90b107d5a89f7bfdd8fa5ad4fe8ff24f650ad89","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:71137c463cc3ab0952fd0a3849bca3c4809b7156416681200f689e255754f7d1","evidence_path":"evidence/20260720T175837Z-50770d44/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:3ed5721ffd5709e1e5687e7b7919628dfad67caedddb362a0041a4f086ce9227","evidence_path":"evidence/20260720T175837Z-50770d44/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:b040d6236b89e30c80ed07303f48d22fea000a693cfe30865110baf2e252755d","evidence_path":"evidence/20260720T175837Z-50770d44/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:985ed74d82b0218e9122b2893a456205fd46e4bf2ab8a6d977a840a9cea84933","evidence_path":"evidence/20260720T175837Z-50770d44/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:a181d248247c8e9e332f72856c96384b43486612a40b37caa041a0a40ffd2382","evidence_path":"evidence/20260720T175837Z-50770d44/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:80c29d3f67bb419cba55252f095bcd474a6dba59258e40f7d027a72558543991","evidence_path":"evidence/20260720T175837Z-50770d44/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:913c0bea2b10546c12a6df8c1ab1561f7e9c2e951746ef2bcb06a343dc044adf","evidence_path":"evidence/20260720T175837Z-50770d44/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:c93394fc03a514e47a8754bb4901a8bf8d576db5a677b2754b9a768e51947b4e","evidence_path":"evidence/20260720T175837Z-50770d44/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:d2b516d2173db7e81b0f34898a13b351e98232e6316a581b98c808333c366c6e","evidence_path":"evidence/20260720T175837Z-50770d44/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'options'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:d056669143d4d8e34ebbcf0c5f49da93eb62d17661be531c367d1fff0f680071","evidence_path":"evidence/20260720T175837Z-50770d44/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:b3aea41e2f5619a798dddb7e182f62045af7a73f05da692bc303b117fc61f206","evidence_path":"evidence/20260720T175837Z-50770d44/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:f913c075ffa8487954cb8ef3c219c96dc2b202ebaa0c86c251b339a44934c3cf","evidence_path":"evidence/20260720T175837Z-50770d44/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:6f407e6f78e6dff8d52dbbe1f5a09d2f50701062209dbb4f643e98011ef44eb4","evidence_path":"evidence/20260720T175837Z-50770d44/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:9debe3a5d9f2f921514ccc362a2b143d18b4cbf87956c4ed69508cc63feb094c","evidence_path":"evidence/20260720T175837Z-50770d44/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:d5400f0c8b49f1f245ddd6f46c585506be8d48296f255d55a893cae2827c5246","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:23d21889f41b6379aa3810e24cab6d1428cdbb09cd278f0217d133b391eaba2a","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:e1ff45c65d8450654d944202632fcc759e4a7a040f176e9edd568f882fe3ce19","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:b3dd6f305f7f27cb5f4cbefd1f886e1a412d077641f0e8a7ea9800844146a1d9","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:ab87905560feeffba8e7afa4c79e48b0707a67cf2aa98ca9cbae57efc374bde2","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-nested-arguments.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:61b53603fb02d3d0a801a9ebe9c9db102b782a5d73a804fbc60368052501d2b5","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:d4065217eb597dcaeb68213257b7dca2577aa735a85cd99e01716db71abcb8ae","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:1fc2479b3f400db5c6abc3423397d6e9f76ad8ce3f9eec1ba145f9ed2388710f","evidence_path":"evidence/20260720T175837Z-50770d44/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:f6458413b7b01df9bd2520ee7ae4d8c72bb54141001fa63a98e78bff0644b899","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:081ae7ae75025e91ccf4d783cc21dc2253896a07c4663e87f64f13c682701537","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:21361f8e3a392acb0382186fddcfe42af68df3b953d6f99b5fccd5ad94ff4724","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:8143c6c8376340526e2f28e294191d22e89f1f5a9c3ebbc2ace1f7dca6d7a086","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:58fa57d7ed212d45be0518ccaaa03e0774c964b0eabd7c5d713c89ae82a50100","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:ef22b39431aba5927f46bdeeef597d999d13e60657c994c2c7294d5e3de41ee6","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:0537bafa16af4c14264cd157ed17dd72a80dd3be370cb0fbdcbc1e64e01e2293","evidence_path":"evidence/20260720T175837Z-50770d44/tool-choice-required-empty.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"fail"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":29,"timestamp":"2026-07-20T17:58:37Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:bdbd181c33f2ed1b31c972991882db3cf4d192569092138a7d29e973cd9debe8","sha256":"sha256:a8cc1361f3145dc01f6d77c6c82c9116b9ffe3c97b34716fe20418455876c40e","source_id":"qwen3:14b","source_kind":"ollama"},"canonical_id":"Qwen/Qwen3-14B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen3:14b","endpoint_id":"qwen3:14b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen3-14b.json","run_id":"20260720T181628Z-e5f2111e","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:f0627700fa623a5a7521bd290619672318fd82016ee9bff0999b340fde6c221a","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:7d13eab8109e6ad56168055eb8793292d36b855292b096d07eeb7f435c898447","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:30e80e88dc24e9b728f3a7a6571d59542337332137c4f72059992eaba00e87b7","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-inventory-order.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:841fe4e31f140e19995e41cbc3ddd99a9545bcad6b31e2a37f47a35f271b902b","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":"turn 1: missing expected tool call: 'get_temperature'","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:0023fa98923c9709ae1e43ceec8ebb6492000868d92c414023d025836286e38a","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:372cd89412f1cc241759d11eb81b694649bd2e532e657b2d5f25b3e6d0901f39","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:24b12719852b00e7afdb356de1d8e147edc5c75528e3a52461498e778b47fbe7","evidence_path":"evidence/20260720T181628Z-e5f2111e/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:9928da236b94994aa6caeb3cd534f164cae5b66cbad644abbbd09f91d9d4f7b7","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:2cc13b313af06578679a975bd87115232dcf31c5d50008b372051aa94172b283","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:c329d587f9fa9cafab0b74966caaf763f327fb987a20a0645d3f13672e483084","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:c6d2f6e3b6d8e7bee2dd50d4c46555ba5f47b1ea2135d5a53df7ff0d575b9bd2","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:ed9044d4758e9933684a5a63012b7cfb7c27b3349cebce6138058997d525fe2b","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:5f3724257abecbce95f37135d572cca23d05067e1b347393d878aef8fd69ee03","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:23281194d55442e2caa41f523e3e19120ed7de7ed8bef1669907bb17afd4077b","evidence_path":"evidence/20260720T181628Z-e5f2111e/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:f2bb74f7186335dd18bd4ea5b25b853d1c21cbf00a37934571b15e30540d4617","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:ec06a175aeaab6f9e8031d4836c9b2875737bc4f761cfcd1498696a8bbcc22ca","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:5892b509394dcaab7111493ce81a9a1966ecaf70c9e8aab4dab827c41aeba0ee","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:98a531d8a72e3f1d06a491c0457089048d2a1e864456a66eece87b59dfce4d2b","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:7f66be8214fa9f98773f696f674c3ecb333b690855db6b056c54645c6d7f2de6","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:bd4091197ad1cb4013fce48bf0b1a66aeeb686f3cbf0e3e93845a1549d539e5f","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:b3f826788b60a41579f6bf4c056d7e58fd32d15043e0f3ee0a265407c03078d3","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:a168aa793ce60794ff5d0162a45c312381ab09855f6d845af81a49b4c9aa31fc","evidence_path":"evidence/20260720T181628Z-e5f2111e/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:2338928c1c740d56fda2fbe47939e24d2e8af0feab5fb4fc36ffecc194105735","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-array-tags.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:467bdbaf9b96b2c1fe24b0b908586fb90df392c0315553379bab622812cfb656","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:49ceaa962087a60bbf0b7c4ce515f293f2b17f02f3ff25ef60a20e025d73de23","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:e4b2e1efedfe3ae8b61e2968e0bb0a816a4613604c5b483e0a9c44487297ba6d","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:c607af39c5c0e335a7459e435b9741d19f496dfafadb121305387f58bb100bb4","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:f476f06b70cb3a3117c625eed704fd391ee060001a83df882ac291b71ad05d30","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:420a1841eeb69d775360b9f1f773f9c5f988f1a3c667dcc63a621f7ed95807c7","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:e69df8d257847eb9b287bf91ee228e06b35d573829366ca808afaa1df5e0bda3","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:5c145808ced71221850b7dd0079ffa6a5fc5091ce351db4694d5d0b6d2be3805","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:1a27253dd49ff263763c6cac3bf91192ed2084a1db7f7453e14dcd173b545d04","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:5725d00da1546f204e9eeb87242a3afdf1f89c64d157b7bf5b094f0f073482be","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:1fe01a6ba0455bb81af729f43e4077109f708a3c759e457244935f1178c4f267","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:632d072c35299ae311e6635f410b229e29b51042b2fd0fad97fd9c676b09d681","evidence_path":"evidence/20260720T181628Z-e5f2111e/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:c5a59c61b1f534ce7491fe069c4ccd9c01f7264a970dd68783d9c34b1da9db6d","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:5891ac92b1e3b48bebbb5958cad2cf0c2cd2fde6a2e3701ea79628d41612a403","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:7fba7886e9f8030c350d32aeb4d979f6b9ce74af2d9e2b4a2e681d469f29b64f","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:be9d7b3c1b84c3f9509b1ee9224f39197326ef4d9cb68349693d7b820591769c","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:38c907da8edd0d5b5685d497468ac59e5f5ce4d8066f7946f478b1a7fef38fd2","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:77e4b531fb419ff2e07f1e36374bbbb4ba7cddc48a81062af8fd2182d3cf49c4","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:d63343dc8a3f311f87dde24ad2706157780601739591683f0c1a807e28830d97","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:1f669cc18749b0321085670c9840d6a06d48113f3acf36c3833f0e70b2a8b892","evidence_path":"evidence/20260720T181628Z-e5f2111e/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:8dfcee3a7e014cff59d631f2cc4b5b01d8f1f9bfbdbfc8c931757c09673dc989","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:b5afbffe593adff8160372ead98f199a17aaee1d5507f4df5ac3ff3efc5d58a0","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:debc25ef3cbf9cad103be7d0631415ef066c93f70a420caa3fa9b8a1a73e1edd","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:4d2893b2b6d5388009457264c226ad40d21d59bdc043e7ea5e60586d689ca536","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:5cc7ffc5307617c2932b5eb9925b19eefec9a5745cddba83dde6b17db235a11d","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:5851de8722e2a3e785afa443ef57ac8802577ce50bfbf576f535cba313e09c82","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:90e1304dd822290877895d8f97820ecdd0c24e838dab83e26c988093cc71fb52","evidence_path":"evidence/20260720T181628Z-e5f2111e/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":30,"timestamp":"2026-07-20T18:16:28Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:359d7dd4bcdab3d86b87d73ac27966f4dbb9f5efdfcc75d34a8764a09474fae7","sha256":"sha256:3e4cb14174460404e7a233e531675303b2fbf7749c02f91864fe311ab6344e4f","source_id":"qwen3:4b","source_kind":"ollama"},"canonical_id":"Qwen/Qwen3-4B-Thinking-2507","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen3:4b","endpoint_id":"qwen3:4b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen3-4b.json","run_id":"20260720T175942Z-d525ca29","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:19bc0fbfc52bdf6cbfa43c49d63e0817344752c5d7319abb62b02a15897ef853","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-calendar-followup.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:f642649638220385abb66c9b9f20948e458f84ec3536bbb2aef6e56b57c4a425","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-flight-booking.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:54dc8a64e33045c9c669c71136b19378f3f37dc3697167790526e5c2b6d1488c","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-inventory-order.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:2c72a0af2a75fb791842d62c8eb2092c152fa10af45a7e55bfbadd41c9631428","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-parallel-followup.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:58ab510adee3ec8b1c01129aeb67329bb5fc079aebeaa70b3fe614a028fe2fbf","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-route.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:4e7cfbe4280ab4d44a0434c5b55c1f1d6b2d94af2a41ad77aa162a177691622b","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-ticket-update.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:c7bff8de08b21f2d9a4f938bcea2aeac67737e3ba6983cf34afb734d874822c8","evidence_path":"evidence/20260720T175942Z-d525ca29/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:9a69b9bd2e6ac3f6dd5932c6556391f9df91bfef8bdbbafaaecbe7b4ee6e24f4","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:da5aceb563dae04f4324cf367bc57554a392792c096ce127f062fc24945f5351","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:58a9908059cb26237a4677ba424245c03a4a6447ec7516ed94ee875acbd53586","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:dc10065c3eb9bd61b6d18eb290228cdd2e8f2a7cf503521b5d8edfefe9279391","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:2080669b638c2eeac569220580ae65598d1d5a4e4e978315d4378e3a2afb7fce","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:6c6a9122b5cd6b4388d310faa402664d17b1f7c9d4c2fedce66e90fbd1e13d30","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:d30a82fa02717f7226c15f67feecbef6a55ca093ac64f4f17f74c328b7a14a83","evidence_path":"evidence/20260720T175942Z-d525ca29/negative-unicode-argument.json","failure":null,"failure_class":"empty_response","failure_reason":"empty response: no content and no tool call","id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"fail"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:6677b7558182e39d89ea163c7e3f8dfcf14c8a378533b0c29e80fa810c70c6ad","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:18a11bbf21c2c3907704c5dc4f8533b48e6cfe79c341392195c6e55b3d9792ef","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:10b738f7bff70e2f4d380392835c821c64afa79c1af173e26af89f841e436c5d","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:946e3aaa4eb0bc472df5775f5a5eb41ceeb718b3a788ca1624e314928356766e","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:5601d006784096ea4bf906aec2f92b3b01c3d20470f640be27aff0db8f3ad1f7","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:4c3ff525c5d2da266a9b39d941135d77dcc25fca752452af6cc1f7ce455640f4","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:8adc45dda76d191198994c2e8935a86d59963c3ffa0d306ace9c86d0a660fc21","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:0f05cacbb1fd3d7320edae6b76e15c3b7a2e01d4d43e689cc367862983028d4c","evidence_path":"evidence/20260720T175942Z-d525ca29/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:93078c2b6690fbf46e3081b5ceda45731b3ccec41591166c7c8758dfba6b5c27","evidence_path":"evidence/20260720T175942Z-d525ca29/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:35c3d808da4090e4fb1eac1cea94113ba3bef0e96905fcfc1dad2452930dcbca","evidence_path":"evidence/20260720T175942Z-d525ca29/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:e78dd73fdd79cc8422108a78d5fb4154f67d025afe79707e10cfd9033d370054","evidence_path":"evidence/20260720T175942Z-d525ca29/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:c7b47530bd5881e6489c8a36f37fb3befef874a9870fe984358241c1ede8943d","evidence_path":"evidence/20260720T175942Z-d525ca29/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:58caf54d0e1393e0e7536b84c307c5983c7cf3f339c0501ba7b370d777428e3e","evidence_path":"evidence/20260720T175942Z-d525ca29/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:90c62f06a129c8cb14886f46bc4907ec67c9300d709f770ae7d28b980d18b0c3","evidence_path":"evidence/20260720T175942Z-d525ca29/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:beda7fa8b17a89b27830d6abff6bec53a94aacecbde3f4ff72e03d3cdbd87c6a","evidence_path":"evidence/20260720T175942Z-d525ca29/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:e3b9877164877f65271a8116095223def74160c87dc09e63fbd3f2f0d5b37296","evidence_path":"evidence/20260720T175942Z-d525ca29/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:14f03ce338b2364349b6740f39187b154d5ddc5286623ca3fa1c0d269d2452a8","evidence_path":"evidence/20260720T175942Z-d525ca29/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:0841529586476cfc7212436630b48f6bd23c9d9b344b6d32312129d7d8e3d85a","evidence_path":"evidence/20260720T175942Z-d525ca29/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:23a195ad29e3d7012bd1473c9d9a0e10f2d5b600331fdb7134a3cef51d610773","evidence_path":"evidence/20260720T175942Z-d525ca29/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:329a56ed77181f05c8fc135afacaa3c668df78654cd5e754b6128a2e0a4a3a70","evidence_path":"evidence/20260720T175942Z-d525ca29/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:7cdeb6568fd2bc89508f6db661014d8ccf218a70bce902d4f2aad80163623128","evidence_path":"evidence/20260720T175942Z-d525ca29/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:e6d5f055530cbc49fb2930dc11534421344c1dc045cab564a55179e3b158176b","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:106501e8d495dff6556a785dfe465447e5271a4a0d5dd361c7cf90f3b11cb37f","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:f5ce8953b48052d69f45ae481645138484b6af38f0d1b8340b5d503a7bdf444d","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:53c7fa45fe0ea3588ccf7180f198a2f9796e35f10874e004467e7be73877b0be","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:d5766b174c335cc9da57c60141741c1650955f8f11dacc992a833d38d316decf","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:627cdf3c99d11dfce9a8e67ff73d165937050796011b099798af48e5088ba4c1","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:b4f3acdaa14c087f77dff84f74c88b42339279f9982043510f926005c1c857d8","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:2015b019da3ea8138bf5df9d24ac2636258cb591e76db369395ad29a9a83cc01","evidence_path":"evidence/20260720T175942Z-d525ca29/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:0450866abb6930fdce200defaef7acb5d14fb7e812af8492438ee0688b98b620","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:1819b24f23b30edd0b27f48fd1511dc027113733f7b54f768a00dcdef16ca5fd","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:66cf3301d03daa56e467f3acba4b4f94051c5f0b32db1413be4ca4f6f16e8baa","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:362eecdce1c4cd3ec30aa12ab7898a6a981cb22662c18f3b7cf271a3863ffed3","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:c74c87138988f1c766b7dab4c516cff089cad95cb5fdd84c0a53ec293fd4a776","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:c0059ee72f1befcaf2f2d225b426c7346ca66688fe10dd6090950cbadd8708bb","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:540732f4296d4dd691e2ad8dd7008e28a8499e1a36af09c8233fffc00e2a42e2","evidence_path":"evidence/20260720T175942Z-d525ca29/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":31,"timestamp":"2026-07-20T17:59:42Z","willitcall_version":"0.1.0"},{"arm_fingerprint":null,"artifact":{"format":"ollama_blob","quantization":{"bits":null,"label":"Q4_K_M","scheme":null},"revision":"sha256:500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41","sha256":"sha256:a3de86cd1c132c822487ededd47a324c50491393e6565cd14bafa40d0b8e686f","source_id":"qwen3:8b","source_kind":"ollama"},"canonical_id":"Qwen/Qwen3-8B","corpus":{"id":"wic-50","revision":"v1","scenario_count":50,"scoring_version":"v1","sha256":"sha256:042f625c2f898787834524323862c42ab1dffc66fe012d27927944705159f9d8"},"display_name":"qwen3:8b","endpoint_id":"qwen3:8b","environment":{"accelerator":null,"architecture":null,"display_label":"Apple M4 Max, 64GB; macOS 26.5.2","memory_bytes":null,"os_name":null,"os_version":null},"family_id":null,"identity_status":"declared","parameter_count_b":null,"preflight_ignored_ports":null,"preflight_override":null,"replication":null,"replication_count":1,"result_file":"ollama-qwen3-8b.json","run_id":"20260720T180246Z-53903d2d","sampling":{"max_tokens":1024,"seed":42,"temperature":0.0,"top_p":1.0},"scenarios":[{"category":"multi_turn","cause":null,"description":"Use a calendar lookup result in a follow-up event call.","evidence_hash":"sha256:2d7675e1a015172b05316c57d819c29b824e4e8280ea6bf4ef9645ed4abab332","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-calendar-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-calendar-followup","rationale":"This asserts using a calendar result in a later create_event call. The date and title are literal in the prompt, and the time is copied verbatim from the prior tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a flight search result in a follow-up booking call.","evidence_hash":"sha256:c92fdcabea8711b96ddcb46fb93fe5c6f4dcaf1e924279ef5d6c7c446d3bf14f","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-flight-booking.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'book_flight'","id":"multi-turn-flight-booking","rationale":"This asserts chaining a flight search into a booking. The route and date are literal in the prompt, while the opaque flight id is copied verbatim from the prior tool-result JSON.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Use an inventory result in a follow-up order call.","evidence_hash":"sha256:fb27d706b2771e922a5af410034a4f5fd6aeef66d4f87962053be405cefea38d","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-inventory-order.json","failure":null,"failure_class":"empty_response","failure_reason":"turn 1: empty response: no content and no tool call","id":"multi-turn-inventory-order","rationale":"This asserts checking availability before placing an order. The opaque SKU and exact quantity are literal in the prompt, and the prior tool-result JSON establishes that stock is available.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Feed two parallel results into one follow-up call.","evidence_hash":"sha256:be3e09c800c03844d8f9c2bbde552debd67386fb30c31011355767fc6f644a95","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-parallel-followup.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-parallel-followup","rationale":"This asserts combining two parallel lookup results in one follow-up call; the city order is defined by the prompt and schema descriptions, and the values come verbatim from tool-result JSON. Turn 1 expects only lookups because the temperatures are not known yet, so an early comparison would invent them.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a geocode result in a follow-up route call.","evidence_hash":"sha256:257300cd520baa8593a5a1f39ac3c8d79270bd4e1b2d9f4bd11e993a21cabd4e","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-route.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-route","rationale":"This asserts passing exact geocode coordinates into a follow-up route call. Turn 1 ignores arguments because a correct model may geocode Fenway Park, Boston, MA, while turn 2 is pinned verbatim by the tool-result JSON.","retried":false,"status":"pass"},{"category":"multi_turn","cause":null,"description":"Use a ticket lookup result in a follow-up update call.","evidence_hash":"sha256:5a3495a9e0277de00906ec49be3d9c00d7955bf3381d5bd411548a5e3c6aa11d","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-ticket-update.json","failure":null,"failure_class":null,"failure_reason":"turn 1: unexpected extra tool call: 'set_ticket_priority'","id":"multi-turn-ticket-update","rationale":"This asserts looking up a ticket before updating its priority; EXT-55 and high are literal in the prompt, and ticket-902 comes verbatim from the tool-result JSON. Turn 1 expects only the lookup because emitting the update before receiving the internal id would require inventing it.","retried":false,"status":"fail"},{"category":"multi_turn","cause":null,"description":"Complete a two-hop chain across three tool calls.","evidence_hash":"sha256:2528b2f0d4f4b97099db63e6ab686865201581b5672ee9c048e920c5fb8ca592","evidence_path":"evidence/20260720T180246Z-53903d2d/multi-turn-two-hop-chain.json","failure":null,"failure_class":null,"failure_reason":null,"id":"multi-turn-two-hop-chain","rationale":"This asserts a three-call dependency chain. The email is literal in the prompt, and each opaque downstream id must be copied verbatim from the immediately preceding tool-result JSON, so no correct alternative value exists.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for simple arithmetic.","evidence_hash":"sha256:95ea9256a084f82fd3de2fa1525e2ec40378f3b8b09c7511df55769abd3334da","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-auto-arithmetic.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-arithmetic","rationale":"This asserts abstaining from tool use for arithmetic. The prompt explicitly says to answer without a tool, and the only offered tool is topically irrelevant weather lookup, so every correct response has no tool call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call an irrelevant tool for a general knowledge reply.","evidence_hash":"sha256:2a54d808b5cbace74b335e84a43b9af1e922fb5b09016f056899e904b221a3d4","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-auto-knowledge.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-auto-knowledge","rationale":"This asserts abstaining from tool use for general knowledge. The prompt explicitly says not to use a tool, and the only offered order lookup cannot answer the ocean question, so every correct response has no call.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Do not call a tool for a simple greeting.","evidence_hash":"sha256:fa74b69055591864c8066d09aa96a269a469d1564d22c3d21168b085f335aa56","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-greeting.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-greeting","rationale":"This asserts honoring a no-tools request for a greeting. The API request enforces tool_choice none, so tool calls are forbidden and the unscored wording of the short text reply cannot cause a false red.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Refuse a tool call when the requested value is outside its schema.","evidence_hash":"sha256:926d6253f6766a8bd323ba457f3d4c6c1bc80cf7bf30721075cfdfe88d71e6b0","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-invalid-schema.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-invalid-schema","rationale":"This asserts refusing to fabricate a schema-valid call for an unsupported value. Kelvin is absent from the celsius-or-fahrenheit enum, and the prompt explicitly requires plain text without a call when the schema cannot express it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve one long ASCII argument exactly.","evidence_hash":"sha256:6335fa3d6e39a15f62e8d3aa1c28b8def0018cd3392ef1e1fdbd3b0fde923cfd","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-long-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-long-argument","rationale":"This asserts exact transport of the 256-character token supplied verbatim in the prompt. The named tool choice fixes the tool, and the run-path max_tokens default is large enough that a correct model is not truncated.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Use a plain text reply instead of an unrelated tool.","evidence_hash":"sha256:7d172bb0ba7336508e8970105bf87c09cb37bd8de026e2f286f701ae6d95b682","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-plain-text.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-plain-text","rationale":"This asserts choosing plain text for an explanatory question. The prompt explicitly says not to send email, and the only offered tool sends email rather than answering the question, so a correct response cannot call it.","retried":false,"status":"pass"},{"category":"negative_trap","cause":null,"description":"Preserve a non-ASCII city argument.","evidence_hash":"sha256:0e8451ac87c84c05db2548f6199a0f6327459862584958eb500f00faf6427c5e","evidence_path":"evidence/20260720T180246Z-53903d2d/negative-unicode-argument.json","failure":null,"failure_class":null,"failure_reason":null,"id":"negative-unicode-argument","rationale":"This asserts copying the exact city spelling supplied verbatim in the prompt. The scorer compares strings in NFC and uses subset matching for the optional country code, so an NFD-emitting model or one adding that optional field is not failed.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the time tool for two cities in parallel.","evidence_hash":"sha256:da72df19fa471ca4ef821cd33f4aee54f19e1e9bd6669f3e2fe273ec2974ab46","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-city-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-city-time","rationale":"This asserts emitting both city-time calls in one response, so a model that can only call sequentially genuinely lacks the measured capability. Boston and Tokyo are literal prompt values that pin both arguments.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools in parallel.","evidence_hash":"sha256:7c0a202460de35eb19a9bebda514d76e8945406e7b68c2c020b4657c52a094c9","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-different-tools","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red. The prompt maps the literal Oslo and Cairo values to distinct named tools, leaving no argument ambiguity.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same tool three times in parallel.","evidence_hash":"sha256:2d85eb9badd5bf69cdd72a84be222e31d0a028a00fa709f2278e4f9efff6ef93","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-three-cities.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-cities","rationale":"This asserts emitting three weather calls in one response, so a model that calls them only sequentially genuinely lacks the capability. Austin, Dublin, and Seoul are supplied verbatim and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call three different tools in parallel.","evidence_hash":"sha256:858c2b81ebdfe56b3daffde4540b630442beb9d097621ed21474352864f04078","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-three-different-tools.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-three-different-tools","rationale":"This asserts emitting three distinct tool calls in one response, so sequential-only calling is a true red. The prompt explicitly pairs Rome with weather, Nairobi with time, and the literal amount and currency codes with conversion.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same arithmetic tool twice in parallel.","evidence_hash":"sha256:5a8cab5a1ad5adcc58b9311903448648a139aa5996a8daa0c6e9908f68f367b3","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-two-calculations.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-calculations","rationale":"This asserts emitting both division calls in one response, so sequential-only calling is a true red. Division is non-commutative, and the named dividend and divisor parameters plus the literal prompt wording force operand position.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call two different tools with multiple arguments.","evidence_hash":"sha256:be26b4d5d582d2a630d985ffa7eae276dc89ca4a62965212a3b2da7715ac50ba","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-two-multiple-args.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-multiple-args","rationale":"This asserts emitting the room and car calls together in one response, so sequential-only calling is a true red. Each room, city, date, and hour is supplied verbatim and mapped to a distinct named tool by the prompt.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call the same search tool twice with distinct queries.","evidence_hash":"sha256:b4dd8068796142c03141cebddfe7b3f6f4b6754d36205cb36ac09ac68ef2381e","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-two-searches.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-two-searches","rationale":"This asserts emitting two search calls in one response, so a model that only calls sequentially genuinely lacks the capability. Both exact query strings appear verbatim in quotes and the prompt explicitly requests separate searches.","retried":false,"status":"pass"},{"category":"parallel_calls","cause":null,"description":"Call weather and time tools for the same city.","evidence_hash":"sha256:e974535c929c85c2ced56440c93acc210d8ff9f6c9fab45ce7b83fa8760a940b","evidence_path":"evidence/20260720T180246Z-53903d2d/parallel-weather-and-time.json","failure":null,"failure_class":null,"failure_reason":null,"id":"parallel-weather-and-time","rationale":"This asserts emitting weather and time calls together in one response, so sequential-only calling is a true red even under auto choice. The prompt explicitly requests both tools and supplies Denver verbatim for each argument.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an array of strings.","evidence_hash":"sha256:8380fc65403f5c30ff34793f401de582d6cb43e1693041224e0b940ad2443891","evidence_path":"evidence/20260720T180246Z-53903d2d/single-array-tags.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-array-tags","rationale":"This asserts an exact string-array argument for document doc-17. Array comparison is positional, and the prompt supplies every literal tag and says in that order, so the expected sequence is fully pinned.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a boolean argument.","evidence_hash":"sha256:417ccc0762992c64fe726a053c044cb0852226ff2c06120f7cef502d01545b1c","evidence_path":"evidence/20260720T180246Z-53903d2d/single-boolean-flag.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-boolean-flag","rationale":"This asserts encoding a disabled state as the boolean false. The opaque feature name audit-log is supplied verbatim, and the imperative Disable pins the boolean without relying on a paraphrasable text answer.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with multiple required arguments.","evidence_hash":"sha256:f32b7a1d4fee0f13e5d9d47ace02df9376e50cb0cdeb309b5e28926672f57f4e","evidence_path":"evidence/20260720T180246Z-53903d2d/single-date-range.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-date-range","rationale":"This asserts populating all fields of an inclusive date-range call. Denver and both ISO dates are supplied verbatim, while from and through map unambiguously to the named start_date and end_date parameters.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with a decimal number.","evidence_hash":"sha256:d43671ff326faad42dcf623fb758c30fb346d404cb311b10c20740f906443d17","evidence_path":"evidence/20260720T180246Z-53903d2d/single-decimal-price.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-decimal-price","rationale":"This asserts preserving a decimal numeric argument in a price call. The opaque SKU AX-9 and exact value 19.95 are supplied verbatim in the prompt, so neither field admits a legitimate rewording.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call a function that accepts no arguments.","evidence_hash":"sha256:b4b557ed8a4f1d941a0079c1682faec971f5657def54a478f1f8292d8ea28059","evidence_path":"evidence/20260720T180246Z-53903d2d/single-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-empty-arguments","rationale":"This asserts calling an empty-argument function. The API request enforces required tool use, only get_service_status is offered, and its object schema defines no properties, so the empty argument object is forced.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an enum argument.","evidence_hash":"sha256:152e399e90cd6d390c26a2bea86412445384b545b6d2ead3da76f8622965f34f","evidence_path":"evidence/20260720T180246Z-53903d2d/single-enum-format.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-enum-format","rationale":"This asserts selecting a schema enum value alongside an opaque report id. The prompt supplies quarterly-7 and csv verbatim, and csv is the exact supported enum spelling, leaving no valid alternate serialization.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an integer argument.","evidence_hash":"sha256:2d4c96a88274a10332f712af6c170ff83d062f3c8322f7df7b8768ef831e4829","evidence_path":"evidence/20260720T180246Z-53903d2d/single-integer-limit.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-integer-limit","rationale":"This asserts encoding an exact integer limit in a mailbox call. The prompt names the support mailbox and says exactly 12, so both expected values are literal and no open-ended text is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with nested option arguments.","evidence_hash":"sha256:17d96de2decd46f9e4e723d6bb854b084beda0212de7dc35fee3aaea60bc21e3","evidence_path":"evidence/20260720T180246Z-53903d2d/single-nested-options.json","failure":null,"failure_class":null,"failure_reason":"schema violation: missing required field 'options'","id":"single-nested-options","rationale":"This asserts constructing the required nested options object with numeric and boolean fields. The quoted prompt, dimensions, and enabled transparency state supply every expected value explicitly, so the nested structure follows directly from the schema.","retried":false,"status":"fail"},{"category":"single_call","cause":null,"description":"Omit an optional argument that the prompt does not provide.","evidence_hash":"sha256:8023c464b3c5fe6d5572922aff8c6f3fb52578b30de6214ad01e03c0ac8c5f0a","evidence_path":"evidence/20260720T180246Z-53903d2d/single-optional-omitted.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-omitted","rationale":"This asserts omitting an optional schema field rather than inventing it. The prompt gives the exact search query and explicitly says not to set a result limit, so the exact expected object is complete.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Include an optional argument when explicitly provided.","evidence_hash":"sha256:ac3f2dbd82862e6d36b204562ee1408d3a74d895a2c81af9c8ffdff5377e5e97","evidence_path":"evidence/20260720T180246Z-53903d2d/single-optional-present.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-optional-present","rationale":"This asserts including an optional limit when the user supplies it. The prompt gives brass compass verbatim as the search query and says exactly 4 results, which pins both expected arguments.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with an ordered array argument.","evidence_hash":"sha256:08fca81846b9125a74dc8db2ba1fcdc295aea8f08522c567b2c0d307cca4061d","evidence_path":"evidence/20260720T180246Z-53903d2d/single-string-list.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-string-list","rationale":"This asserts preserving an ordered array of opaque track ids. Array comparison is positional, and the prompt supplies the playlist name and every id verbatim while saying in that order.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one weather tool with a city argument.","evidence_hash":"sha256:c771d1338f1c509f7d3564348a69d7d0d3db69339ee911ddc115d8c6323c59ce","evidence_path":"evidence/20260720T180246Z-53903d2d/single-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-weather","rationale":"This asserts selecting the weather tool with one city argument. Boston appears verbatim in the prompt, the schema requires only city, and no generated natural-language answer is compared.","retried":false,"status":"pass"},{"category":"single_call","cause":null,"description":"Call one tool with zero and a negative number.","evidence_hash":"sha256:cf2efdecd5881f2c76f23fcd5f7ac7c228735b105754b92bc4caca8b4e6116c0","evidence_path":"evidence/20260720T180246Z-53903d2d/single-zero-and-negative.json","failure":null,"failure_class":null,"failure_reason":null,"id":"single-zero-and-negative","rationale":"This asserts preserving zero and a negative decimal in distinct numeric fields. The prompt says from -12.5 to 0, and the named minimum and maximum parameters force which literal occupies each position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an array argument.","evidence_hash":"sha256:c537f66aee9a6f18eafdca6362a5fde96e56bb92b5b4e16d5f40f27c79086e3f","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-array-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-array-arguments","rationale":"This asserts reconstructing an ordered array from streamed arguments. Array comparison is positional, the prompt supplies all opaque record ids verbatim and says in that order, and the API request enforces the named tool.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with a boolean argument.","evidence_hash":"sha256:fdafb66674ca8cbc39c617ab2ce91566aa7389004b24667d0e32db156c6439fe","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-boolean-argument.json","failure":null,"failure_class":null,"failure_reason":"wrong value for 'service': expected billing, got billing service","id":"streaming-boolean-argument","rationale":"This asserts reconstructing a boolean argument from a streamed tool call. The prompt supplies billing verbatim and the imperative Enable pins enabled to true, so no paraphrasable response text is scored.","retried":false,"status":"fail"},{"category":"streaming","cause":null,"description":"Stream a call to a function with no arguments.","evidence_hash":"sha256:3f9b78ac8a902336560bae02d741290a64a7695d3a8814a96eec2c1a5dc43675","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-empty-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-empty-arguments","rationale":"This asserts reconstructing an empty argument object from a streamed call. The API request requires tool use, get_uptime is the only offered tool, and its schema has no properties, so the expected call is forced.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with an enum argument.","evidence_hash":"sha256:addd15b7605fdb3dc0e8fa5661e92350f3acaa3c3ba799969c4dc0569411266c","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-enum-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-enum-arguments","rationale":"This asserts reconstructing an enum argument from streamed fragments. The prompt supplies payments and warn verbatim, and warn exactly matches a supported schema enum value, leaving no valid alternative.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream a tool call with nested arguments.","evidence_hash":"sha256:40d2a638c198f647c3a56d27cb86d2af7f81f71a76dbed9f437c7764edbe1e60","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-nested-arguments.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-nested-arguments","rationale":"This asserts reconstructing a nested object from streamed argument fragments. The opaque marker name and both signed coordinates are literal prompt values, and the schema uniquely assigns x and y within position.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream three calls to the same tool.","evidence_hash":"sha256:69e58a5fd1f9cc4b27217cd0c9f3fcb91c79b72992acd14e8332a717bbcbbf64","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-parallel-three.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-three","rationale":"This asserts reconstructing three parallel calls from one streamed response, so sequential-only calling is a true red. Accra, Kyoto, and Quito are supplied verbatim, and the prompt explicitly requests three calls.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream two different tool calls.","evidence_hash":"sha256:1510a95dcd780932b5ca9dc601fedb323098ba1bafa3ff72f4a02f0680d3ea3c","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-parallel-two.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-parallel-two","rationale":"This asserts reconstructing two parallel calls from one streamed response, so sequential-only calling is a true red. The prompt explicitly maps literal Prague to weather and Manila to time, pinning both tool-argument pairs.","retried":false,"status":"pass"},{"category":"streaming","cause":null,"description":"Stream one weather tool call.","evidence_hash":"sha256:8d8797cbfe04b1d1cc2749ee62d969058dc9f59f7f78600630b7b9b35ffdfaf6","evidence_path":"evidence/20260720T180246Z-53903d2d/streaming-weather.json","failure":null,"failure_class":null,"failure_reason":null,"id":"streaming-weather","rationale":"This asserts reconstructing a basic tool call from streamed output. The API request enforces the named get_weather tool, and the prompt supplies Seattle verbatim as its only required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice when a tool is clearly needed.","evidence_hash":"sha256:8ab750dbb427d2c1e6f8374eeffdfa1c3bd6e805ea665a96fe2fd8d651153226","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-auto-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-call","rationale":"This asserts choosing a needed lookup under auto tool choice, where the API leaves call-or-text discretion to the model. The requested temperature requires the only offered tool, and Helsinki is supplied verbatim as its city argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Use auto tool choice without calling an irrelevant tool.","evidence_hash":"sha256:51a637041db6b27a4f9b2ee010415baf54a4ca99417823c44937f7e05fbc0980","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-auto-no-call.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-auto-no-call","rationale":"This asserts declining an irrelevant tool under auto choice, where the API leaves call-or-text discretion to the model. The prompt requests literal text and the only tool fetches temperatures, while response wording is not scored.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force one named function from two available tools.","evidence_hash":"sha256:560fb71c5c126b3c6d38d2535ff01cddaa1d34039f94911fa7fa333b5408b9c7","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-named.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named","rationale":"This asserts honoring a named tool choice among two tools. The API request enforces search_contacts rather than leaving tool selection to the model, and the opaque name Ada Stone is supplied verbatim for the required argument.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Force a named empty-argument function.","evidence_hash":"sha256:85ec20bafc7e8e864ca7cfb8814744ea3e146a73a72376d667b17f2f119e7a81","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-named-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-named-empty","rationale":"This asserts honoring a named tool choice when another tool is offered. The API request enforces current_region, and its schema has no properties, so the model has no discretion over the tool or arguments.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Forbid tool calls for a direct text response.","evidence_hash":"sha256:a969b86a5b5699104515bcd98552afa990717060eddc32fc819ad205e0d0cb6e","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-none.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-none","rationale":"This asserts honoring tool_choice none. The API request forbids every offered tool call, so the model has discretion only over its direct text response, whose contents are not compared by the scorer.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require one tool call with explicit arguments.","evidence_hash":"sha256:d2f0592712b74999aecb11db057ab15c021c95ad0c42e93197b0aae49d477bab","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-required.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required","rationale":"This asserts honoring required tool choice with an explicit argument. The API request enforces a call, lookup_order is the only offered tool, and opaque order id ORD-204 appears verbatim in the prompt.","retried":false,"status":"pass"},{"category":"tool_choice_modes","cause":null,"description":"Require a call to an empty-argument function.","evidence_hash":"sha256:1715731bb97452281dacb9e95f225da8093df447ce86f16acc8b1cb896be308a","evidence_path":"evidence/20260720T180246Z-53903d2d/tool-choice-required-empty.json","failure":null,"failure_class":null,"failure_reason":null,"id":"tool-choice-required-empty","rationale":"This asserts honoring required tool choice with an empty schema. The API request enforces at least one call, current_sequence is the only offered tool, and its schema permits no arguments, removing model discretion.","retried":false,"status":"pass"}],"schema_version":3,"server":{"chat_template":null,"decode_mode_source":"preset_mapping","effective_decode_mode":"unconstrained_post_hoc","launch_config_sha256":null,"preset_name":"ollama","quirk_flags":[],"recorded_decode_mode":"unknown","reported_version":"0.32.1"},"stack_index":32,"timestamp":"2026-07-20T18:02:46Z","willitcall_version":"0.1.0"}]}