{"title":"Agent tool selection, November 2026","page":"https://soxoa.com/research/agent-tool-selection-2026-11","measuredOn":"2026-11-16","what":"How often a model picks the expected tool, given one request and the real MCP tool definitions of one Soxoa product next to synthetic competing tools. Selection only; completion is not measured.","settings":{"runsPerRequestPerModel":5,"temperature":1,"maxOutputTokens":1024,"toolChoice":"auto","toolOrder":"shuffled, seeded by product, request and run, identical for every model","models":["openai/gpt-6.1-terra","anthropic/claude-haiku-5.5","google/gemini-3.8-flash"],"modelHost":"OpenRouter"},"intervals":"low and high are 95% Wilson intervals that treat every run as independent and are too narrow; correctRequestInterval resamples whole requests and is the one to trust.","baseline":{"date":"2026-11-16","variant":"live","products":{"postedly":{"product":"postedly","requests":15,"tools":16,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.85,"low":0.7388517410884038,"high":0.9190265075770431,"k":51,"n":60},"correctRequestInterval":{"low":0.6333333333333333,"high":1,"requests":12},"selections":{"product_tool":56,"decoy":4},"argsValidAmongCalls":{"rate":1,"low":0.9398260695220669,"high":0.9999999999999999,"k":60,"n":60},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9299509861127319,"high":0.9999999999999999,"k":51,"n":51}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":5,"chosen":{"postedly_prepare_document":5},"selections":{"product_tool":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":1,"chosen":{"address_book_search":4,"postedly_contacts":1},"selections":{"decoy":4,"product_tool":1}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5},"postedly_contacts":{"address_book_search":4}},"chosenDecoys":{"address_book_search":4},"chosenOtherProduct":{"postedly_services":5},"costUsd":0.518038,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8166666666666667,"low":0.7008000677353371,"high":0.8944231096286387,"k":49,"n":60},"correctRequestInterval":{"low":0.6,"high":1,"requests":12},"selections":{"product_tool":54,"nothing":4,"decoy":2},"argsValidAmongCalls":{"rate":1,"low":0.9358038555118846,"high":1,"k":56,"n":56},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9272997032640949,"high":0.9999999999999999,"k":49,"n":49}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":1,"chosen":{"(nothing)":4,"postedly_prepare_document":1},"selections":{"nothing":4,"product_tool":1}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":3,"chosen":{"address_book_search":2,"postedly_contacts":3},"selections":{"decoy":2,"product_tool":3}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"(nothing)":4},"postedly_contacts":{"address_book_search":2}},"chosenDecoys":{"address_book_search":2},"chosenOtherProduct":{"postedly_services":5},"costUsd":0.091972,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.5833333333333334,"low":0.45727854313239624,"high":0.6993591351212818,"k":35,"n":60},"correctRequestInterval":{"low":0.3333333333333333,"high":0.8333333333333334,"requests":12},"selections":{"product_tool":50,"nothing":5,"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9010957324106112,"high":1,"k":35,"n":35}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":0,"chosen":{"address_book_search":5},"selections":{"decoy":5}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":0,"chosen":{"postedly_contacts":2,"postedly_drafts":3},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"postedly_drafts":5},"postedly_save_contact":{"(nothing)":5},"postedly_contacts":{"address_book_search":5},"postedly_save_draft":{"postedly_contacts":2,"postedly_drafts":3}},"chosenDecoys":{"address_book_search":5},"chosenOtherProduct":{"postedly_services":5,"postedly_drafts":8,"postedly_contacts":2},"costUsd":0.370519,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"75b7a1b04a7a5807de59a956ca8f27a4ace74491","files":["server/mcp.ts","server/saved-correspondence.ts"],"filesModifiedSinceCommit":false}}},"portravo":{"product":"portravo","requests":14,"tools":10,"decoys":9,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.34545454545454546,"low":0.23362533364154048,"high":0.4774634198933736,"k":19,"n":55},"correctRequestInterval":{"low":0.10909090909090909,"high":0.6181818181818182,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.8318156346315496,"high":1,"k":19,"n":19}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":1,"chosen":{"get_profile":4,"get_takeoff":1},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":3,"chosen":{"export_reviewed_estimate":3,"get_profile":2},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"list_projects":{"get_profile":5},"list_projects | get_takeoff_status":{"get_profile":5},"create_project | preflight_takeoff":{"get_profile":5},"run_takeoff":{"get_profile":5},"get_takeoff":{"get_profile":4},"export_reviewed_estimate":{"get_profile":2},"preflight_takeoff":{"get_profile":5},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_profile":31},"costUsd":0.230171,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.7454545454545455,"low":0.6169787660766216,"high":0.8418802724267501,"k":41,"n":55},"correctRequestInterval":{"low":0.4909090909090909,"high":0.9454545454545454,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9143295511310926,"high":1,"k":41,"n":41}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":4,"chosen":{"get_profile":1,"preflight_takeoff":4},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":2,"chosen":{"get_profile":3,"preflight_takeoff":2},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"create_project | preflight_takeoff":{"get_profile":1},"run_takeoff":{"preflight_takeoff":5},"preflight_takeoff":{"get_profile":3},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_profile":4,"preflight_takeoff":5},"costUsd":0.043576,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.7272727272727273,"low":0.5976755051758511,"high":0.8271939749198636,"k":40,"n":55},"correctRequestInterval":{"low":0.45454545454545453,"high":1,"requests":11},"selections":{"product_tool":51,"decoy":4},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9123754607496077,"high":1,"k":40,"n":40}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":0,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"list_cloud_files":4,"preflight_takeoff":1},"selections":{"decoy":4,"product_tool":1}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"create_project | preflight_takeoff":{"list_pricing_markets":5},"run_takeoff":{"preflight_takeoff":5},"attach_plan_pdf":{"list_cloud_files":4,"preflight_takeoff":1}},"chosenDecoys":{"list_cloud_files":4},"chosenOtherProduct":{"list_pricing_markets":5,"preflight_takeoff":6},"costUsd":0.200258,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"2a87d1456e5c3ba5ea9719b6e72a30f70e724701","files":["src/lib/mcp/tools.ts"],"filesModifiedSinceCommit":false}}},"intakra":{"product":"intakra","requests":15,"tools":15,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8333333333333334,"low":0.7196814850279217,"high":0.9068692279867897,"k":50,"n":60},"correctRequestInterval":{"low":0.5833333333333334,"high":1,"requests":12},"selections":{"product_tool":52,"nothing":8},"argsValidAmongCalls":{"rate":1,"low":0.9312054095871178,"high":1,"k":52,"n":52},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"i01","kind":"product","expected":["get_daily_brief"],"n":5,"correct":5,"chosen":{"get_daily_brief":5},"selections":{"product_tool":5}},{"request_id":"i02","kind":"product","expected":["prioritize_accounts","list_opportunities"],"n":5,"correct":5,"chosen":{"prioritize_accounts":5},"selections":{"product_tool":5}},{"request_id":"i03","kind":"product","expected":["get_account_changes"],"n":5,"correct":5,"chosen":{"get_account_changes":5},"selections":{"product_tool":5}},{"request_id":"i04","kind":"product","expected":["get_account"],"n":5,"correct":5,"chosen":{"get_account":5},"selections":{"product_tool":5}},{"request_id":"i05","kind":"product","expected":["add_accounts"],"n":5,"correct":5,"chosen":{"add_accounts":5},"selections":{"product_tool":5}},{"request_id":"i06","kind":"product","expected":["get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i07","kind":"product","expected":["set_offering","get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i08","kind":"product","expected":["get_best_contact"],"n":5,"correct":5,"chosen":{"get_best_contact":5},"selections":{"product_tool":5}},{"request_id":"i09","kind":"product","expected":["request_accounts"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"i10","kind":"product","expected":["list_suggestions"],"n":5,"correct":5,"chosen":{"list_suggestions":5},"selections":{"product_tool":5}},{"request_id":"i11","kind":"product","expected":["get_project_updates"],"n":5,"correct":5,"chosen":{"get_project_updates":5},"selections":{"product_tool":5}},{"request_id":"i12","kind":"product","expected":["score_buying_signal_evidence"],"n":5,"correct":0,"chosen":{"(nothing)":3,"get_offering":2},"selections":{"nothing":3,"product_tool":2}},{"request_id":"i13","kind":"control","expected":["linkedin_people_search"],"n":5,"correct":5,"chosen":{"linkedin_people_search":5},"selections":{"decoy":5}},{"request_id":"i14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"i15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"request_accounts":{"(nothing)":5},"score_buying_signal_evidence":{"(nothing)":3,"get_offering":2}},"chosenDecoys":{},"chosenOtherProduct":{"get_offering":2},"costUsd":0.331428,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8833333333333333,"low":0.7782097940014187,"high":0.9423235259654995,"k":53,"n":60},"correctRequestInterval":{"low":0.6833333333333333,"high":1,"requests":12},"selections":{"product_tool":58,"nothing":2},"argsValidAmongCalls":{"rate":1,"low":0.9378800031047062,"high":0.9999999999999999,"k":58,"n":58},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9324156955469234,"high":1,"k":53,"n":53}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"i01","kind":"product","expected":["get_daily_brief"],"n":5,"correct":5,"chosen":{"get_daily_brief":5},"selections":{"product_tool":5}},{"request_id":"i02","kind":"product","expected":["prioritize_accounts","list_opportunities"],"n":5,"correct":5,"chosen":{"prioritize_accounts":5},"selections":{"product_tool":5}},{"request_id":"i03","kind":"product","expected":["get_account_changes"],"n":5,"correct":5,"chosen":{"get_account_changes":5},"selections":{"product_tool":5}},{"request_id":"i04","kind":"product","expected":["get_account"],"n":5,"correct":5,"chosen":{"get_account":5},"selections":{"product_tool":5}},{"request_id":"i05","kind":"product","expected":["add_accounts"],"n":5,"correct":5,"chosen":{"add_accounts":5},"selections":{"product_tool":5}},{"request_id":"i06","kind":"product","expected":["get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i07","kind":"product","expected":["set_offering","get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i08","kind":"product","expected":["get_best_contact"],"n":5,"correct":5,"chosen":{"get_best_contact":5},"selections":{"product_tool":5}},{"request_id":"i09","kind":"product","expected":["request_accounts"],"n":5,"correct":3,"chosen":{"request_accounts":3,"(nothing)":2},"selections":{"product_tool":3,"nothing":2}},{"request_id":"i10","kind":"product","expected":["list_suggestions"],"n":5,"correct":5,"chosen":{"list_suggestions":5},"selections":{"product_tool":5}},{"request_id":"i11","kind":"product","expected":["get_project_updates"],"n":5,"correct":5,"chosen":{"get_project_updates":5},"selections":{"product_tool":5}},{"request_id":"i12","kind":"product","expected":["score_buying_signal_evidence"],"n":5,"correct":0,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i13","kind":"control","expected":["linkedin_people_search"],"n":5,"correct":5,"chosen":{"linkedin_people_search":5},"selections":{"decoy":5}},{"request_id":"i14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"i15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"request_accounts":{"(nothing)":2},"score_buying_signal_evidence":{"get_offering":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_offering":5},"costUsd":0.061786,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8333333333333334,"low":0.7196814850279217,"high":0.9068692279867897,"k":50,"n":60},"correctRequestInterval":{"low":0.5833333333333334,"high":1,"requests":12},"selections":{"product_tool":58,"decoy":1,"nothing":1},"argsValidAmongCalls":{"rate":1,"low":0.938868520215908,"high":1,"k":59,"n":59},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"i01","kind":"product","expected":["get_daily_brief"],"n":5,"correct":5,"chosen":{"get_daily_brief":5},"selections":{"product_tool":5}},{"request_id":"i02","kind":"product","expected":["prioritize_accounts","list_opportunities"],"n":5,"correct":5,"chosen":{"prioritize_accounts":5},"selections":{"product_tool":5}},{"request_id":"i03","kind":"product","expected":["get_account_changes"],"n":5,"correct":5,"chosen":{"get_account_changes":5},"selections":{"product_tool":5}},{"request_id":"i04","kind":"product","expected":["get_account"],"n":5,"correct":5,"chosen":{"get_account":5},"selections":{"product_tool":5}},{"request_id":"i05","kind":"product","expected":["add_accounts"],"n":5,"correct":5,"chosen":{"add_accounts":5},"selections":{"product_tool":5}},{"request_id":"i06","kind":"product","expected":["get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i07","kind":"product","expected":["set_offering","get_offering"],"n":5,"correct":5,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i08","kind":"product","expected":["get_best_contact"],"n":5,"correct":5,"chosen":{"get_best_contact":5},"selections":{"product_tool":5}},{"request_id":"i09","kind":"product","expected":["request_accounts"],"n":5,"correct":0,"chosen":{"get_company_profile":1,"(nothing)":1,"list_suggestions":2,"get_account":1},"selections":{"decoy":1,"nothing":1,"product_tool":3}},{"request_id":"i10","kind":"product","expected":["list_suggestions"],"n":5,"correct":5,"chosen":{"list_suggestions":5},"selections":{"product_tool":5}},{"request_id":"i11","kind":"product","expected":["get_project_updates"],"n":5,"correct":5,"chosen":{"get_project_updates":5},"selections":{"product_tool":5}},{"request_id":"i12","kind":"product","expected":["score_buying_signal_evidence"],"n":5,"correct":0,"chosen":{"get_offering":5},"selections":{"product_tool":5}},{"request_id":"i13","kind":"control","expected":["linkedin_people_search"],"n":5,"correct":5,"chosen":{"linkedin_people_search":5},"selections":{"decoy":5}},{"request_id":"i14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"i15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"request_accounts":{"get_company_profile":1,"(nothing)":1,"list_suggestions":2,"get_account":1},"score_buying_signal_evidence":{"get_offering":5}},"chosenDecoys":{"get_company_profile":1},"chosenOtherProduct":{"list_suggestions":2,"get_account":1,"get_offering":5},"costUsd":0.252655,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"b19d4deb547132c8d9bc8c932cff34f63ff3f917","files":["src/app/api/mcp/tools.ts","src/lib/public-mcp/core.ts"],"filesModifiedSinceCommit":false}}},"mentionedon":{"product":"mentionedon","requests":15,"tools":13,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8,"low":0.6821795390818299,"high":0.8817161026314104,"k":48,"n":60},"correctRequestInterval":{"low":0.5666666666666667,"high":0.9833333333333333,"requests":12},"selections":{"product_tool":48,"decoy":6,"nothing":6},"argsValidAmongCalls":{"rate":1,"low":0.9335841332189981,"high":1,"k":54,"n":54},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9258973488472576,"high":0.9999999999999999,"k":48,"n":48}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_website_fixes"],"n":5,"correct":0,"chosen":{"generate_schema_markup":5},"selections":{"decoy":5}},{"request_id":"m09","kind":"product","expected":["verify_website_fixes","list_monitored_businesses"],"n":5,"correct":4,"chosen":{"list_monitored_businesses":4,"seo_audit_page":1},"selections":{"product_tool":4,"decoy":1}},{"request_id":"m10","kind":"product","expected":["summarize_scan_evidence"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":5,"chosen":{"analyze_discovery_readiness":5},"selections":{"product_tool":5}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":4,"chosen":{"build_visibility_action_plan":4,"(nothing)":1},"selections":{"product_tool":4,"nothing":1}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_website_fixes":{"generate_schema_markup":5},"verify_website_fixes | list_monitored_businesses":{"seo_audit_page":1},"summarize_scan_evidence":{"(nothing)":5},"build_visibility_action_plan":{"(nothing)":1}},"chosenDecoys":{"generate_schema_markup":5,"seo_audit_page":1},"chosenOtherProduct":{},"costUsd":0.423653,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7,"low":0.5749105303360957,"high":0.801019897472731,"k":42,"n":60},"correctRequestInterval":{"low":0.4666666666666667,"high":0.9166666666666666,"requests":12},"selections":{"product_tool":45,"decoy":9,"nothing":6},"argsValidAmongCalls":{"rate":1,"low":0.9335841332189981,"high":1,"k":54,"n":54},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9161983874908382,"high":1,"k":42,"n":42}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_website_fixes"],"n":5,"correct":0,"chosen":{"generate_schema_markup":5},"selections":{"decoy":5}},{"request_id":"m09","kind":"product","expected":["verify_website_fixes","list_monitored_businesses"],"n":5,"correct":1,"chosen":{"list_monitored_businesses":1,"seo_audit_page":4},"selections":{"product_tool":1,"decoy":4}},{"request_id":"m10","kind":"product","expected":["summarize_scan_evidence"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":4,"chosen":{"analyze_discovery_readiness":4,"list_monitored_businesses":1},"selections":{"product_tool":5}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":2,"chosen":{"list_monitored_businesses":2,"build_visibility_action_plan":2,"(nothing)":1},"selections":{"product_tool":4,"nothing":1}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_website_fixes":{"generate_schema_markup":5},"verify_website_fixes | list_monitored_businesses":{"seo_audit_page":4},"summarize_scan_evidence":{"(nothing)":5},"analyze_discovery_readiness":{"list_monitored_businesses":1},"build_visibility_action_plan":{"list_monitored_businesses":2,"(nothing)":1}},"chosenDecoys":{"generate_schema_markup":5,"seo_audit_page":4},"chosenOtherProduct":{"list_monitored_businesses":3},"costUsd":0.07561,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7,"low":0.5749105303360957,"high":0.801019897472731,"k":42,"n":60},"correctRequestInterval":{"low":0.45,"high":0.9166666666666666,"requests":12},"selections":{"product_tool":48,"decoy":7,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9161983874908382,"high":1,"k":42,"n":42}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_website_fixes"],"n":5,"correct":0,"chosen":{"generate_schema_markup":5},"selections":{"decoy":5}},{"request_id":"m09","kind":"product","expected":["verify_website_fixes","list_monitored_businesses"],"n":5,"correct":3,"chosen":{"list_monitored_businesses":3,"seo_audit_page":2},"selections":{"product_tool":3,"decoy":2}},{"request_id":"m10","kind":"product","expected":["summarize_scan_evidence"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":4,"chosen":{"analyze_discovery_readiness":4,"list_monitored_businesses":1},"selections":{"product_tool":5}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":0,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_website_fixes":{"generate_schema_markup":5},"verify_website_fixes | list_monitored_businesses":{"seo_audit_page":2},"summarize_scan_evidence":{"(nothing)":5},"analyze_discovery_readiness":{"list_monitored_businesses":1},"build_visibility_action_plan":{"list_monitored_businesses":5}},"chosenDecoys":{"generate_schema_markup":5,"seo_audit_page":2},"chosenOtherProduct":{"list_monitored_businesses":6},"costUsd":0.353055,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"f6eaa2dacd872e184beaadda8696c5012c4bda53","files":["lib/mcp/catalog.ts","lib/mcp/account-tools.ts","lib/mcp/schemas.ts"],"filesModifiedSinceCommit":false}}}}},"schemaFixRerun":{"note":"Postedly and Portravo tool definitions re-snapshotted after the flat-schema fix went live. asServed uses the current descriptions; descriptionsRestored uses the current schemas with the baseline descriptions put back, to separate the schema change from the description changes made in the same releases. Anthropic requests were sent without removing any schema keyword.","asServed":{"date":"2026-11-16","variant":"live","products":{"postedly":{"product":"postedly","requests":15,"tools":16,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.9,"low":0.7985030550748896,"high":0.9533578005427642,"k":54,"n":60},"correctRequestInterval":{"low":0.7166666666666667,"high":1,"requests":12},"selections":{"product_tool":59,"decoy":1},"argsValidAmongCalls":{"rate":1,"low":0.9398260695220669,"high":0.9999999999999999,"k":60,"n":60},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9335841332189981,"high":1,"k":54,"n":54}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":5,"chosen":{"postedly_prepare_document":5},"selections":{"product_tool":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":4,"chosen":{"postedly_contacts":4,"address_book_search":1},"selections":{"product_tool":4,"decoy":1}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5},"postedly_contacts":{"address_book_search":1}},"chosenDecoys":{"address_book_search":1},"chosenOtherProduct":{"postedly_services":5},"costUsd":0.500527,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7833333333333333,"low":0.6637973165265748,"high":0.8687707895359299,"k":47,"n":60},"correctRequestInterval":{"low":0.5666666666666667,"high":1,"requests":12},"selections":{"product_tool":52,"nothing":5,"decoy":3},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9244398288016114,"high":1,"k":47,"n":47}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":2,"chosen":{"address_book_search":3,"postedly_contacts":2},"selections":{"decoy":3,"product_tool":2}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"(nothing)":5},"postedly_contacts":{"address_book_search":3}},"chosenDecoys":{"address_book_search":3},"chosenOtherProduct":{"postedly_services":5},"costUsd":0.0943,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.5833333333333334,"low":0.45727854313239624,"high":0.6993591351212818,"k":35,"n":60},"correctRequestInterval":{"low":0.3333333333333333,"high":0.8333333333333334,"requests":12},"selections":{"product_tool":50,"nothing":5,"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9010957324106112,"high":1,"k":35,"n":35}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":0,"chosen":{"address_book_search":5},"selections":{"decoy":5}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":0,"chosen":{"postedly_contacts":4,"postedly_drafts":1},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"postedly_drafts":5},"postedly_save_contact":{"(nothing)":5},"postedly_contacts":{"address_book_search":5},"postedly_save_draft":{"postedly_contacts":4,"postedly_drafts":1}},"chosenDecoys":{"address_book_search":5},"chosenOtherProduct":{"postedly_services":5,"postedly_drafts":6,"postedly_contacts":4},"costUsd":0.382012,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"3c6094d530b667ac59e835bb349e3eea5ddc665d","files":["server/mcp.ts","server/saved-correspondence.ts"],"filesModifiedSinceCommit":false}}},"portravo":{"product":"portravo","requests":14,"tools":10,"decoys":9,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.8181818181818182,"low":0.6966928866143076,"high":0.898124385519693,"k":45,"n":55},"correctRequestInterval":{"low":0.5454545454545454,"high":1,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9213457380593594,"high":1,"k":45,"n":45}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"run_takeoff":{"preflight_takeoff":5},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"preflight_takeoff":5},"costUsd":0.248944,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.8181818181818182,"low":0.6966928866143076,"high":0.898124385519693,"k":45,"n":55},"correctRequestInterval":{"low":0.5454545454545454,"high":1,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9213457380593594,"high":1,"k":45,"n":45}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"run_takeoff":{"preflight_takeoff":5},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"preflight_takeoff":5},"costUsd":0.046103,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.7454545454545455,"low":0.6169787660766216,"high":0.8418802724267501,"k":41,"n":55},"correctRequestInterval":{"low":0.4727272727272727,"high":1,"requests":11},"selections":{"product_tool":51,"decoy":3,"nothing":1},"argsValidAmongCalls":{"rate":1,"low":0.9335841332189981,"high":1,"k":54,"n":54},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9143295511310926,"high":1,"k":41,"n":41}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":1,"chosen":{"preflight_takeoff":1,"list_pricing_markets":4},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"list_cloud_files":3,"(nothing)":1,"get_takeoff_status":1},"selections":{"decoy":3,"nothing":1,"product_tool":1}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"create_project | preflight_takeoff":{"list_pricing_markets":4},"run_takeoff":{"preflight_takeoff":5},"attach_plan_pdf":{"list_cloud_files":3,"(nothing)":1,"get_takeoff_status":1}},"chosenDecoys":{"list_cloud_files":3},"chosenOtherProduct":{"list_pricing_markets":4,"preflight_takeoff":5,"get_takeoff_status":1},"costUsd":0.21276,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"live","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"cf5ef412850de2d0a612aee4e2d234fd337648a8","files":["src/lib/mcp/tools.ts"],"filesModifiedSinceCommit":false}}}}},"descriptionsRestored":{"date":"2026-11-16","variant":"proposed","products":{"postedly":{"product":"postedly","requests":15,"tools":16,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.8666666666666667,"low":0.7583460540992789,"high":0.9308597302169036,"k":52,"n":60},"correctRequestInterval":{"low":0.6833333333333333,"high":1,"requests":12},"selections":{"product_tool":58,"nothing":1,"decoy":1},"argsValidAmongCalls":{"rate":1,"low":0.938868520215908,"high":1,"k":59,"n":59},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9312054095871178,"high":1,"k":52,"n":52}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":3,"chosen":{"postedly_services":1,"(nothing)":1,"postedly_prepare_document":3},"selections":{"product_tool":4,"nothing":1}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":4,"chosen":{"postedly_contacts":4,"address_book_search":1},"selections":{"product_tool":4,"decoy":1}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":6,"(nothing)":1},"postedly_contacts":{"address_book_search":1}},"chosenDecoys":{"address_book_search":1},"chosenOtherProduct":{"postedly_services":6},"costUsd":0.510459,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7666666666666667,"low":0.6456348865960799,"high":0.8556056838156894,"k":46,"n":60},"correctRequestInterval":{"low":0.5333333333333333,"high":1,"requests":12},"selections":{"product_tool":51,"nothing":5,"decoy":4},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9229238226702192,"high":1,"k":46,"n":46}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":5,"chosen":{"postedly_save_contact":5},"selections":{"product_tool":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":1,"chosen":{"address_book_search":4,"postedly_contacts":1},"selections":{"decoy":4,"product_tool":1}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":5,"chosen":{"postedly_save_draft":5},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"(nothing)":5},"postedly_contacts":{"address_book_search":4}},"chosenDecoys":{"address_book_search":4},"chosenOtherProduct":{"postedly_services":5},"costUsd":0.094109,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.5833333333333334,"low":0.45727854313239624,"high":0.6993591351212818,"k":35,"n":60},"correctRequestInterval":{"low":0.3333333333333333,"high":0.8333333333333334,"requests":12},"selections":{"product_tool":50,"nothing":5,"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9010957324106112,"high":1,"k":35,"n":35}},"none":{"runs":15,"requests":3,"correct":{"rate":1,"low":0.7961107336956521,"high":1,"k":15,"n":15},"correctRequestInterval":{"low":1,"high":1,"requests":3},"selections":{"nothing":15},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":0,"requests":0,"correct":null,"correctRequestInterval":null,"selections":{},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null}},"perRequest":[{"request_id":"p01","kind":"product","expected":["postedly_services"],"n":5,"correct":5,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p02","kind":"product","expected":["postedly_products"],"n":5,"correct":5,"chosen":{"postedly_products":5},"selections":{"product_tool":5}},{"request_id":"p03","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_services":5},"selections":{"product_tool":5}},{"request_id":"p04","kind":"product","expected":["postedly_quote","postedly_prepare_document"],"n":5,"correct":0,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p05","kind":"product","expected":["postedly_orders"],"n":5,"correct":5,"chosen":{"postedly_orders":5},"selections":{"product_tool":5}},{"request_id":"p06","kind":"product","expected":["postedly_order_status","postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_order_status":5},"selections":{"product_tool":5}},{"request_id":"p07","kind":"product","expected":["postedly_delivery_report"],"n":5,"correct":5,"chosen":{"postedly_delivery_report":5},"selections":{"product_tool":5}},{"request_id":"p08","kind":"product","expected":["postedly_save_contact"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p09","kind":"product","expected":["postedly_contacts"],"n":5,"correct":0,"chosen":{"address_book_search":5},"selections":{"decoy":5}},{"request_id":"p10","kind":"product","expected":["postedly_save_draft"],"n":5,"correct":0,"chosen":{"postedly_contacts":2,"postedly_drafts":3},"selections":{"product_tool":5}},{"request_id":"p11","kind":"product","expected":["postedly_drafts"],"n":5,"correct":5,"chosen":{"postedly_drafts":5},"selections":{"product_tool":5}},{"request_id":"p12","kind":"product","expected":["postedly_checkout"],"n":5,"correct":5,"chosen":{"postedly_checkout":5},"selections":{"product_tool":5}},{"request_id":"p13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"p15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"postedly_quote | postedly_prepare_document":{"postedly_services":5,"postedly_drafts":5},"postedly_save_contact":{"(nothing)":5},"postedly_contacts":{"address_book_search":5},"postedly_save_draft":{"postedly_contacts":2,"postedly_drafts":3}},"chosenDecoys":{"address_book_search":5},"chosenOtherProduct":{"postedly_services":5,"postedly_drafts":8,"postedly_contacts":2},"costUsd":0.378659,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"proposed","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"3c6094d530b667ac59e835bb349e3eea5ddc665d","files":["server/mcp.ts","server/saved-correspondence.ts"],"filesModifiedSinceCommit":false}}},"portravo":{"product":"portravo","requests":14,"tools":10,"decoys":9,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.34545454545454546,"low":0.23362533364154048,"high":0.4774634198933736,"k":19,"n":55},"correctRequestInterval":{"low":0.12727272727272726,"high":0.6,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.8318156346315496,"high":1,"k":19,"n":19}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":1,"chosen":{"get_profile":4,"list_projects":1},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":1,"chosen":{"get_profile":4,"get_takeoff":1},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":2,"chosen":{"get_profile":3,"export_reviewed_estimate":2},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"list_projects":{"get_profile":5},"list_projects | get_takeoff_status":{"get_profile":4},"create_project | preflight_takeoff":{"get_profile":5},"run_takeoff":{"get_profile":5},"get_takeoff":{"get_profile":4},"export_reviewed_estimate":{"get_profile":3},"preflight_takeoff":{"get_profile":5},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_profile":31},"costUsd":0.23634,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.7090909090909091,"low":0.5785902917965606,"high":0.8122896298914968,"k":39,"n":55},"correctRequestInterval":{"low":0.43636363636363634,"high":0.9454545454545454,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9103301463997611,"high":1,"k":39,"n":39}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":2,"chosen":{"get_profile":3,"preflight_takeoff":2},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":2,"chosen":{"get_profile":3,"preflight_takeoff":2},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"create_project | preflight_takeoff":{"get_profile":3},"run_takeoff":{"preflight_takeoff":5},"preflight_takeoff":{"get_profile":3},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_profile":6,"preflight_takeoff":5},"costUsd":0.044004,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.7454545454545455,"low":0.6169787660766216,"high":0.8418802724267501,"k":41,"n":55},"correctRequestInterval":{"low":0.4727272727272727,"high":1,"requests":11},"selections":{"product_tool":51,"decoy":4},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9143295511310926,"high":1,"k":41,"n":41}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":5,"chosen":{"list_projects":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":1,"chosen":{"preflight_takeoff":1,"list_pricing_markets":4},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":5,"chosen":{"get_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":5,"chosen":{"export_reviewed_estimate":5},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"get_takeoff":1,"list_cloud_files":4},"selections":{"product_tool":1,"decoy":4}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"create_project | preflight_takeoff":{"list_pricing_markets":4},"run_takeoff":{"preflight_takeoff":5},"attach_plan_pdf":{"get_takeoff":1,"list_cloud_files":4}},"chosenDecoys":{"list_cloud_files":4},"chosenOtherProduct":{"list_pricing_markets":4,"preflight_takeoff":5,"get_takeoff":1},"costUsd":0.203905,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"proposed","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"cf5ef412850de2d0a612aee4e2d234fd337648a8","files":["src/lib/mcp/tools.ts"],"filesModifiedSinceCommit":false}}}}},"getProfileOnly":{"note":"Portravo, gpt-6.1-terra only, current schemas and descriptions except get_profile, which has its baseline wording back.","summary":{"date":"2026-11-16","variant":"proposed","products":{"portravo":{"product":"portravo","requests":14,"tools":10,"decoys":9,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":70,"errors":0,"kinds":{"product":{"runs":55,"requests":11,"correct":{"rate":0.32727272727272727,"low":0.21812895688825168,"high":0.45897023823900523,"k":18,"n":55},"correctRequestInterval":{"low":0.09090909090909091,"high":0.6,"requests":11},"selections":{"product_tool":50,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9286499658256813,"high":1,"k":50,"n":50},"argsValidAmongCorrectCalls":{"rate":1,"low":0.8241154494176252,"high":1,"k":18,"n":18}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"t01","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":5,"chosen":{"preflight_takeoff":5},"selections":{"product_tool":5}},{"request_id":"t02","kind":"product","expected":["list_pricing_markets"],"n":5,"correct":5,"chosen":{"list_pricing_markets":5},"selections":{"product_tool":5}},{"request_id":"t03","kind":"product","expected":["list_projects"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t04","kind":"product","expected":["list_projects","get_takeoff_status"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t05","kind":"product","expected":["create_project","preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t06","kind":"product","expected":["run_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t07","kind":"product","expected":["get_takeoff"],"n":5,"correct":2,"chosen":{"get_takeoff":2,"get_profile":3},"selections":{"product_tool":5}},{"request_id":"t08","kind":"product","expected":["export_reviewed_estimate"],"n":5,"correct":1,"chosen":{"get_profile":4,"export_reviewed_estimate":1},"selections":{"product_tool":5}},{"request_id":"t09","kind":"product","expected":["get_profile"],"n":5,"correct":5,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t10","kind":"product","expected":["preflight_takeoff"],"n":5,"correct":0,"chosen":{"get_profile":5},"selections":{"product_tool":5}},{"request_id":"t11","kind":"product","expected":["attach_plan_pdf"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t12","kind":"control","expected":["get_weather_forecast"],"n":5,"correct":5,"chosen":{"get_weather_forecast":5},"selections":{"decoy":5}},{"request_id":"t13","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"t14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"list_projects":{"get_profile":5},"list_projects | get_takeoff_status":{"get_profile":5},"create_project | preflight_takeoff":{"get_profile":5},"run_takeoff":{"get_profile":5},"get_takeoff":{"get_profile":3},"export_reviewed_estimate":{"get_profile":4},"preflight_takeoff":{"get_profile":5},"attach_plan_pdf":{"(nothing)":5}},"chosenDecoys":{},"chosenOtherProduct":{"get_profile":32},"costUsd":0.255828,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"proposed","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"cf5ef412850de2d0a612aee4e2d234fd337648a8","files":["src/lib/mcp/tools.ts"],"filesModifiedSinceCommit":false}}}}}}},"mentionedonRewordingSimulation":{"note":"Proposed wording for MentionedOn tools offered to the same models. A simulation, not the live product.","summary":{"products":{"mentionedon":{"product":"mentionedon","requests":15,"tools":13,"decoys":10,"models":{"openai/gpt-6.1-terra":{"vendor":"OpenAI","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7666666666666667,"low":0.6456348865960799,"high":0.8556056838156894,"k":46,"n":60},"correctRequestInterval":{"low":0.5166666666666667,"high":1,"requests":12},"selections":{"product_tool":46,"decoy":5,"nothing":9},"argsValidAmongCalls":{"rate":1,"low":0.9299509861127319,"high":0.9999999999999999,"k":51,"n":51},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9229238226702192,"high":1,"k":46,"n":46}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_business_schema_and_copy"],"n":5,"correct":0,"chosen":{"generate_schema_markup":5},"selections":{"decoy":5}},{"request_id":"m09","kind":"product","expected":["verify_installed_schema_on_page","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m10","kind":"product","expected":["calculate_mention_rates_from_answers"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":1,"chosen":{"(nothing)":4,"analyze_discovery_readiness":1},"selections":{"nothing":4,"product_tool":1}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":5,"chosen":{"build_visibility_action_plan":5},"selections":{"product_tool":5}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_business_schema_and_copy":{"generate_schema_markup":5},"calculate_mention_rates_from_answers":{"(nothing)":5},"analyze_discovery_readiness":{"(nothing)":4}},"chosenDecoys":{"generate_schema_markup":5},"chosenOtherProduct":{},"costUsd":0.474969,"servedModels":["openai/gpt-6.1-terra"],"temperatures":[1],"sampleErrors":[]},"anthropic/claude-haiku-5.5":{"vendor":"Anthropic","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.7666666666666667,"low":0.6456348865960799,"high":0.8556056838156894,"k":46,"n":60},"correctRequestInterval":{"low":0.5333333333333333,"high":0.9666666666666667,"requests":12},"selections":{"product_tool":51,"decoy":4,"nothing":5},"argsValidAmongCalls":{"rate":1,"low":0.9347128562105721,"high":0.9999999999999999,"k":55,"n":55},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9229238226702192,"high":1,"k":46,"n":46}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_business_schema_and_copy"],"n":5,"correct":0,"chosen":{"generate_schema_markup":4,"list_monitored_businesses":1},"selections":{"decoy":4,"product_tool":1}},{"request_id":"m09","kind":"product","expected":["verify_installed_schema_on_page","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m10","kind":"product","expected":["calculate_mention_rates_from_answers"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":3,"chosen":{"analyze_discovery_readiness":3,"list_monitored_businesses":2},"selections":{"product_tool":5}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":3,"chosen":{"list_monitored_businesses":2,"build_visibility_action_plan":3},"selections":{"product_tool":5}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_business_schema_and_copy":{"generate_schema_markup":4,"list_monitored_businesses":1},"calculate_mention_rates_from_answers":{"(nothing)":5},"analyze_discovery_readiness":{"list_monitored_businesses":2},"build_visibility_action_plan":{"list_monitored_businesses":2}},"chosenDecoys":{"generate_schema_markup":4},"chosenOtherProduct":{"list_monitored_businesses":5},"costUsd":0.080502,"servedModels":["anthropic/claude-haiku-5.5"],"temperatures":[1],"sampleErrors":[]},"google/gemini-3.8-flash":{"vendor":"Google","runs":75,"errors":0,"kinds":{"product":{"runs":60,"requests":12,"correct":{"rate":0.75,"low":0.6276768905451521,"high":0.8422361442158816,"k":45,"n":60},"correctRequestInterval":{"low":0.5,"high":1,"requests":12},"selections":{"product_tool":49,"nothing":8,"decoy":3},"argsValidAmongCalls":{"rate":1,"low":0.9312054095871178,"high":1,"k":52,"n":52},"argsValidAmongCorrectCalls":{"rate":1,"low":0.9213457380593594,"high":1,"k":45,"n":45}},"none":{"runs":10,"requests":2,"correct":{"rate":1,"low":0.7224598312333834,"high":1,"k":10,"n":10},"correctRequestInterval":{"low":1,"high":1,"requests":2},"selections":{"nothing":10},"argsValidAmongCalls":null,"argsValidAmongCorrectCalls":null},"control":{"runs":5,"requests":1,"correct":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"correctRequestInterval":{"low":1,"high":1,"requests":1},"selections":{"decoy":5},"argsValidAmongCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5},"argsValidAmongCorrectCalls":{"rate":1,"low":0.5655085052479191,"high":1,"k":5,"n":5}}},"perRequest":[{"request_id":"m01","kind":"product","expected":["list_monitored_businesses","get_visibility_report"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m02","kind":"product","expected":["get_share_of_voice","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m03","kind":"product","expected":["get_improvement_results","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m04","kind":"product","expected":["get_ai_referrals","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m05","kind":"product","expected":["list_articles","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m06","kind":"product","expected":["get_website_fixes","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m07","kind":"product","expected":["inspect_business_website","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m08","kind":"product","expected":["draft_business_schema_and_copy"],"n":5,"correct":0,"chosen":{"(nothing)":2,"generate_schema_markup":3},"selections":{"nothing":2,"decoy":3}},{"request_id":"m09","kind":"product","expected":["verify_installed_schema_on_page","list_monitored_businesses"],"n":5,"correct":5,"chosen":{"list_monitored_businesses":5},"selections":{"product_tool":5}},{"request_id":"m10","kind":"product","expected":["calculate_mention_rates_from_answers"],"n":5,"correct":0,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m11","kind":"product","expected":["analyze_discovery_readiness"],"n":5,"correct":5,"chosen":{"analyze_discovery_readiness":5},"selections":{"product_tool":5}},{"request_id":"m12","kind":"product","expected":["build_visibility_action_plan"],"n":5,"correct":0,"chosen":{"list_monitored_businesses":4,"(nothing)":1},"selections":{"product_tool":4,"nothing":1}},{"request_id":"m13","kind":"control","expected":["keyword_research"],"n":5,"correct":5,"chosen":{"keyword_research":5},"selections":{"decoy":5}},{"request_id":"m14","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}},{"request_id":"m15","kind":"none","expected":["none"],"n":5,"correct":5,"chosen":{"(nothing)":5},"selections":{"nothing":5}}],"confusion":{"draft_business_schema_and_copy":{"(nothing)":2,"generate_schema_markup":3},"calculate_mention_rates_from_answers":{"(nothing)":5},"build_visibility_action_plan":{"list_monitored_businesses":4,"(nothing)":1}},"chosenDecoys":{"generate_schema_markup":3},"chosenOtherProduct":{"list_monitored_businesses":4},"costUsd":0.396819,"servedModels":["google/gemini-3.8-flash"],"temperatures":[1],"sampleErrors":[]}},"meta":{"date":"2026-11-16","variant":"proposed","n":5,"temperature":1,"maxTokens":1024,"source":{"commit":"f6eaa2dacd872e184beaadda8696c5012c4bda53","files":["lib/mcp/catalog.ts","lib/mcp/account-tools.ts","lib/mcp/schemas.ts"],"filesModifiedSinceCommit":false}}}}}},"spendUsd":{"baseline":3.978832,"schemaFixRerun":3.2504},"requestsAndDecoys":{"note":"The request wording and the synthetic competing tools offered next to each product's real tools. Names, addresses and ids in the requests are made up; no tool is ever executed.","products":{"postedly":{"requests":[{"id":"p01","text":"What kinds of mail and fax can I send from here, and roughly what does each start at?","expected":["postedly_services"],"kind":"product"},{"id":"p02","text":"What handwritten card designs can I choose from?","expected":["postedly_products"],"kind":"product"},{"id":"p03","text":"I need to send a letter by certified mail to my landlord, Jordan Sample, 400 Example Avenue, Springfield, IL 62701. What would that cost?","expected":["postedly_quote","postedly_prepare_document"],"kind":"product","note":"A quote or the document upload step are both reasonable first moves."},{"id":"p04","text":"Fax my signed lease to 916-555-0142 for me today.","expected":["postedly_quote","postedly_prepare_document"],"kind":"product","note":"Sending starts with a quote or a document handoff."},{"id":"p05","text":"Where is the certified letter I sent to Sample Mutual Insurance last week?","expected":["postedly_orders"],"kind":"product"},{"id":"p06","text":"Did the fax to Dr. Example's office go through? The order is ORD-48213.","expected":["postedly_order_status","postedly_delivery_report"],"kind":"product"},{"id":"p07","text":"I need a record I can keep showing that my certified letter was delivered. Order ORD-48213.","expected":["postedly_delivery_report"],"kind":"product"},{"id":"p08","text":"Save my sister's address so I can reuse it: Riley Sample, 22 Placeholder Lane, Austin, TX 78701. Call it \"Sister\".","expected":["postedly_save_contact"],"kind":"product"},{"id":"p09","text":"Do I already have an address saved for my accountant?","expected":["postedly_contacts"],"kind":"product"},{"id":"p10","text":"I'm not ready to send this yet, so save it as a draft: a postcard to Sam Sample, 9 Sample Court, Portland, OR 97201, saying \"Wish you were here\".","expected":["postedly_save_draft"],"kind":"product"},{"id":"p11","text":"Pull up the drafts I saved earlier this week.","expected":["postedly_drafts"],"kind":"product"},{"id":"p12","text":"My quote ID is q_demo_123 and I'm ready to pay. Give me the link.","expected":["postedly_checkout"],"kind":"product"},{"id":"p13","text":"Can you proofread this before I mail it? \"Dear Ms. Sample, please accept this letter as formal notice that I will be vacating the apartment on December 31.\"","expected":["none"],"kind":"none"},{"id":"p14","text":"How should I format an address on a letter going to the United Kingdom?","expected":["none"],"kind":"none"},{"id":"p15","text":"Text my landlord and tell him the rent will arrive on Friday.","expected":["none"],"kind":"none","note":"No tool offered sends text messages."}],"decoyTools":[{"name":"send_letter","description":"Mail a printed letter to a postal address using a generic mail API. Takes a recipient address, a return address and a link to a PDF. Choose standard or tracked delivery and color or black and white printing. Returns a letter ID and an expected delivery window.","inputSchema":{"type":"object","properties":{"to":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]},"from":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]},"pdf_url":{"type":"string","format":"uri"},"delivery":{"type":"string","enum":["standard","tracked"]},"color":{"type":"boolean"}},"additionalProperties":false,"required":["to","from","pdf_url"]},"synthetic":true},{"name":"send_fax","description":"Send a fax from a PDF or image URL to a fax number. Returns a fax ID and the transmission status once it is known.","inputSchema":{"type":"object","properties":{"fax_number":{"type":"string","description":"Destination fax number with country code"},"document_url":{"type":"string","format":"uri"},"cover_page_text":{"type":"string","maxLength":500}},"additionalProperties":false,"required":["fax_number","document_url"]},"synthetic":true},{"name":"send_postcard","description":"Print and mail a postcard with a front image and a short message to one address. Standard sizes only. Returns a postcard ID and a preview link.","inputSchema":{"type":"object","properties":{"to":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]},"front_image_url":{"type":"string","format":"uri"},"message":{"type":"string","maxLength":400},"size":{"type":"string","enum":["4x6","6x9"]}},"additionalProperties":false,"required":["to","front_image_url","message"]},"synthetic":true},{"name":"send_greeting_card","description":"Send a physical greeting card for an occasion such as a birthday, thank you or sympathy. Pick an occasion, write a message, and give the recipient address.","inputSchema":{"type":"object","properties":{"occasion":{"type":"string","enum":["birthday","thank_you","sympathy","congratulations","holiday"]},"message":{"type":"string","maxLength":600},"to":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]}},"additionalProperties":false,"required":["occasion","message","to"]},"synthetic":true},{"name":"track_shipment","description":"Look up the delivery status of a parcel or letter from a carrier tracking number. Returns status, last scan location and estimated delivery date.","inputSchema":{"type":"object","properties":{"tracking_number":{"type":"string"},"carrier":{"type":"string","enum":["usps","ups","fedex","dhl"]}},"additionalProperties":false,"required":["tracking_number"]},"synthetic":true},{"name":"verify_address","description":"Check whether a postal address is deliverable and return the standardized form. Does not send anything.","inputSchema":{"type":"object","properties":{"address":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]}},"additionalProperties":false,"required":["address"]},"synthetic":true},{"name":"convert_to_pdf","description":"Convert a document, spreadsheet or image at a URL to PDF and return a link to the PDF. Files up to 20 MB.","inputSchema":{"type":"object","properties":{"file_url":{"type":"string","format":"uri"},"page_size":{"type":"string","enum":["letter","a4"]}},"additionalProperties":false,"required":["file_url"]},"synthetic":true},{"name":"address_book_search","description":"Search the user's address book by name, email or company. Returns up to 20 matching contacts with their saved addresses and phone numbers.","inputSchema":{"type":"object","properties":{"query":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":50}},"additionalProperties":false,"required":["query"]},"synthetic":true},{"name":"address_book_add_contact","description":"Add a person or company to the user's address book with a name, an address and optional phone and email. Returns the new contact ID.","inputSchema":{"type":"object","properties":{"name":{"type":"string"},"address":{"type":"object","properties":{"name":{"type":"string"},"line1":{"type":"string"},"line2":{"type":"string"},"city":{"type":"string"},"state":{"type":"string"},"postal_code":{"type":"string"},"country":{"type":"string","description":"Two-letter country code"}},"additionalProperties":false,"required":["name","line1","city","postal_code"]},"phone":{"type":"string"},"email":{"type":"string","format":"email"}},"additionalProperties":false,"required":["name"]},"synthetic":true},{"name":"print_document","description":"Send a document to the user's office printer. Choose copies, paper size and double-sided printing. Returns a print job ID.","inputSchema":{"type":"object","properties":{"document_url":{"type":"string","format":"uri"},"copies":{"type":"integer","minimum":1,"maximum":100},"paper":{"type":"string","enum":["letter","legal","a4"]},"duplex":{"type":"boolean"}},"additionalProperties":false,"required":["document_url"]},"synthetic":true}]},"portravo":{"requests":[{"id":"t01","text":"How much does a takeoff cost, and which trades can you do automatically?","expected":["preflight_takeoff"],"kind":"product"},{"id":"t02","text":"Which cities or regions do you have construction pricing for?","expected":["list_pricing_markets"],"kind":"product"},{"id":"t03","text":"Show me my recent projects.","expected":["list_projects"],"kind":"product"},{"id":"t04","text":"Is the takeoff on my Maple Street kitchen job finished yet?","expected":["list_projects","get_takeoff_status"],"kind":"product","note":"The project id is not given, so finding the project first is also correct."},{"id":"t05","text":"Set up a new estimate project called Maple Street Kitchen Remodel at 18 Example Road, Sacramento, CA for the electrical and plumbing trades.","expected":["create_project","preflight_takeoff"],"kind":"product"},{"id":"t06","text":"I uploaded the floor plan PDF on the project page. Start the takeoff for project prj_demo01 and spend at most 1 credit.","expected":["run_takeoff"],"kind":"product"},{"id":"t07","text":"What quantities came out of the takeoff for project prj_demo01, and do any lines still need review?","expected":["get_takeoff"],"kind":"product"},{"id":"t08","text":"I've finished reviewing every line. Give me the Excel file for project prj_demo01.","expected":["export_reviewed_estimate"],"kind":"product"},{"id":"t09","text":"Which account am I signed in as?","expected":["get_profile"],"kind":"product"},{"id":"t10","text":"How many takeoff credits do I have left?","expected":["preflight_takeoff"],"kind":"product"},{"id":"t11","text":"Attach the plan PDF I just shared to project prj_demo01.","expected":["attach_plan_pdf"],"kind":"product"},{"id":"t12","text":"What's the weather going to be like in Sacramento on Thursday? We're pouring concrete.","expected":["get_weather_forecast"],"kind":"control","note":"A decoy is the right tool here."},{"id":"t13","text":"What waste factor do people usually add for drywall?","expected":["none"],"kind":"none"},{"id":"t14","text":"Explain what a construction takeoff is in plain English.","expected":["none"],"kind":"none"}],"decoyTools":[{"name":"estimate_material_costs","description":"Estimate the cost of building materials from a list of items with quantities and a ZIP code. Uses regional average retail prices. Returns a line total and a grand total.","inputSchema":{"type":"object","properties":{"items":{"type":"array","minItems":1,"maxItems":100,"items":{"type":"object","properties":{"material":{"type":"string"},"quantity":{"type":"number","minimum":0},"unit":{"type":"string"}},"additionalProperties":false,"required":["material","quantity"]}},"zip_code":{"type":"string","pattern":"^[0-9]{5}$"}},"additionalProperties":false,"required":["items","zip_code"]},"synthetic":true},{"name":"extract_pdf_text","description":"Extract the text and tables from a PDF at a URL. Returns text by page. Does not interpret drawings or measure anything.","inputSchema":{"type":"object","properties":{"pdf_url":{"type":"string","format":"uri"},"pages":{"type":"string","description":"Page range such as 1-5"}},"additionalProperties":false,"required":["pdf_url"]},"synthetic":true},{"name":"list_cloud_files","description":"List files and folders in the user's connected cloud storage. Returns names, sizes and modified dates.","inputSchema":{"type":"object","properties":{"folder":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":200}},"additionalProperties":false},"synthetic":true},{"name":"create_task","description":"Create a task in the user's project management board with a title, due date and optional assignee. Returns the task ID.","inputSchema":{"type":"object","properties":{"title":{"type":"string","maxLength":200},"due_date":{"type":"string","format":"date"},"assignee":{"type":"string"},"notes":{"type":"string","maxLength":2000}},"additionalProperties":false,"required":["title"]},"synthetic":true},{"name":"search_building_codes","description":"Search building code text for a jurisdiction. Returns matching sections with citations. Reference only, not legal advice.","inputSchema":{"type":"object","properties":{"jurisdiction":{"type":"string"},"query":{"type":"string"},"code_year":{"type":"integer","minimum":2000,"maximum":2100}},"additionalProperties":false,"required":["jurisdiction","query"]},"synthetic":true},{"name":"create_spreadsheet","description":"Create an Excel spreadsheet from rows of values and return a download link.","inputSchema":{"type":"object","properties":{"sheet_name":{"type":"string","maxLength":31},"columns":{"type":"array","items":{"type":"string"},"minItems":1},"rows":{"type":"array","items":{"type":"array","items":{"type":["string","number"]}}}},"additionalProperties":false,"required":["columns","rows"]},"synthetic":true},{"name":"get_weather_forecast","description":"Get a daily weather forecast for a city or ZIP code for up to 10 days. Returns temperature, precipitation chance and wind.","inputSchema":{"type":"object","properties":{"location":{"type":"string"},"days":{"type":"integer","minimum":1,"maximum":10}},"additionalProperties":false,"required":["location"]},"synthetic":true},{"name":"measure_floor_plan_image","description":"Estimate room areas from a floor plan image when given the drawing scale. Returns approximate square footage per room. Estimates only.","inputSchema":{"type":"object","properties":{"image_url":{"type":"string","format":"uri"},"scale":{"type":"string","description":"Drawing scale such as 1/4 inch = 1 foot"}},"additionalProperties":false,"required":["image_url","scale"]},"synthetic":true},{"name":"lookup_contractor_license","description":"Look up a contractor license by state and license number. Returns status, classification and expiry date.","inputSchema":{"type":"object","properties":{"state":{"type":"string","minLength":2,"maxLength":2},"license_number":{"type":"string"}},"additionalProperties":false,"required":["state","license_number"]},"synthetic":true}]},"intakra":{"requests":[{"id":"i01","text":"Who should I call today?","expected":["get_daily_brief"],"kind":"product"},{"id":"i02","text":"Of the companies I'm tracking, which look most ready to buy right now?","expected":["prioritize_accounts","list_opportunities"],"kind":"product"},{"id":"i03","text":"Anything new happening at the companies on my list this week?","expected":["get_account_changes"],"kind":"product"},{"id":"i04","text":"What do we know about globex.com?","expected":["get_account"],"kind":"product"},{"id":"i05","text":"Add ramp.com and brex.com to my list.","expected":["add_accounts"],"kind":"product"},{"id":"i06","text":"What does Intakra think we sell?","expected":["get_offering"],"kind":"product"},{"id":"i07","text":"Change what we sell to: payroll software for restaurant groups with 5 to 50 locations.","expected":["set_offering","get_offering"],"kind":"product","note":"Reading the current text first is also correct."},{"id":"i08","text":"Who at the company should I reach about lead eop_demo123, and why them?","expected":["get_best_contact"],"kind":"product"},{"id":"i09","text":"Find more companies like acme.com and globex.com that we could start tracking.","expected":["request_accounts"],"kind":"product"},{"id":"i10","text":"Which companies has Intakra suggested I add?","expected":["list_suggestions"],"kind":"product"},{"id":"i11","text":"What changed on lead eop_demo123 since I last looked?","expected":["get_project_updates"],"kind":"product"},{"id":"i12","text":"Here is what I have on Initech: three job ads for integration engineers posted in October, a new VP of Operations in September, and they are a mid-market SaaS company, which is our ideal customer. Score how strong the buying signals are.","expected":["score_buying_signal_evidence"],"kind":"product"},{"id":"i13","text":"Find the LinkedIn profile of the VP of Sales at Hooli.","expected":["linkedin_people_search"],"kind":"control","note":"A decoy is the right tool here."},{"id":"i14","text":"Briefly, what is the BANT framework for qualifying a lead?","expected":["none"],"kind":"none"},{"id":"i15","text":"In two sentences, what is the difference between inbound and outbound sales?","expected":["none"],"kind":"none"}],"decoyTools":[{"name":"search_companies","description":"Search a company database by industry, employee count, country and keywords. Returns up to 25 companies with domain, size and industry.","inputSchema":{"type":"object","properties":{"industry":{"type":"string"},"employee_min":{"type":"integer","minimum":1},"employee_max":{"type":"integer","minimum":1},"country":{"type":"string"},"keywords":{"type":"array","items":{"type":"string"},"maxItems":10}},"additionalProperties":false},"synthetic":true},{"name":"enrich_contact","description":"Find a work email, job title and phone number for a person from their name and company domain.","inputSchema":{"type":"object","properties":{"full_name":{"type":"string"},"company_domain":{"type":"string"}},"additionalProperties":false,"required":["full_name","company_domain"]},"synthetic":true},{"name":"get_company_news","description":"Return recent news articles that mention a company, newest first, with headline, source and date.","inputSchema":{"type":"object","properties":{"domain":{"type":"string"},"days":{"type":"integer","minimum":1,"maximum":90}},"additionalProperties":false,"required":["domain"]},"synthetic":true},{"name":"crm_find_deals","description":"List deals in the connected CRM by stage or owner. Returns deal name, amount, stage and last activity date.","inputSchema":{"type":"object","properties":{"stage":{"type":"string"},"owner":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":100}},"additionalProperties":false},"synthetic":true},{"name":"crm_create_lead","description":"Create a lead record in the connected CRM with a company name, a contact and optional notes. Returns the lead ID.","inputSchema":{"type":"object","properties":{"company":{"type":"string"},"contact_name":{"type":"string"},"contact_email":{"type":"string","format":"email"},"notes":{"type":"string","maxLength":2000}},"additionalProperties":false,"required":["company"]},"synthetic":true},{"name":"send_email","description":"Send an email from the user's connected mailbox to one recipient.","inputSchema":{"type":"object","properties":{"to":{"type":"string","format":"email"},"subject":{"type":"string","maxLength":200},"body":{"type":"string","maxLength":10000}},"additionalProperties":false,"required":["to","subject","body"]},"synthetic":true},{"name":"linkedin_people_search","description":"Search LinkedIn profiles by job title, company and location. Returns names, titles and profile links.","inputSchema":{"type":"object","properties":{"title":{"type":"string"},"company":{"type":"string"},"location":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":25}},"additionalProperties":false},"synthetic":true},{"name":"get_company_profile","description":"Return firmographic details for a company domain: headquarters, employee range, founding year, funding stage and technologies used.","inputSchema":{"type":"object","properties":{"domain":{"type":"string"}},"additionalProperties":false,"required":["domain"]},"synthetic":true},{"name":"draft_cold_email","description":"Write a first outreach email to a prospect from a company name, the prospect's role and a short description of what the sender sells. Returns a subject and body for the user to review.","inputSchema":{"type":"object","properties":{"company":{"type":"string"},"prospect_role":{"type":"string"},"offer":{"type":"string","maxLength":500},"tone":{"type":"string","enum":["formal","friendly","direct"]}},"additionalProperties":false,"required":["company","prospect_role","offer"]},"synthetic":true},{"name":"web_search","description":"Search the web and return the top results with titles, links and snippets.","inputSchema":{"type":"object","properties":{"query":{"type":"string"},"num_results":{"type":"integer","minimum":1,"maximum":10}},"additionalProperties":false,"required":["query"]},"synthetic":true}]},"mentionedon":{"requests":[{"id":"m01","text":"How does my business show up when people ask ChatGPT for recommendations?","expected":["list_monitored_businesses","get_visibility_report"],"kind":"product"},{"id":"m02","text":"Which websites do the AI assistants cite when they answer questions about my kind of business?","expected":["get_share_of_voice","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m03","text":"Did the page we published last month change what the assistants say about us?","expected":["get_improvement_results","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m04","text":"Are people actually arriving at our website from ChatGPT?","expected":["get_ai_referrals","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m05","text":"Show me the articles you drafted for us.","expected":["list_articles","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m06","text":"What business facts have we already confirmed for our website fixes?","expected":["get_website_fixes","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m07","text":"Go through my website and pull out my business hours, address and services so we can fix what the assistants get wrong.","expected":["inspect_business_website","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m08","text":"Turn these confirmed facts into schema markup for my site: Sample Plumbing, 100 Example Street, Fresno, CA, open Monday to Friday 8 to 5, services are drain cleaning and water heater repair.","expected":["draft_website_fixes"],"kind":"product"},{"id":"m09","text":"I installed the schema on our contact page, https://sampleplumbing.example/contact. Can you check that it is live?","expected":["verify_website_fixes","list_monitored_businesses"],"kind":"product","note":"No business id is given, so reading the connected businesses first (the documented starting point) is also a correct first step."},{"id":"m10","text":"I copied answers from four AI chats for \"best plumber in Fresno\". ChatGPT named Sample Plumbing first. Perplexity named it third. Gemini did not name it. Claude named it second. How often were we mentioned?","expected":["summarize_scan_evidence"],"kind":"product"},{"id":"m11","text":"Score my site for AI readiness. It has a homepage and an about page, no structured data, a Google Business Profile with 40 reviews, and no FAQ page.","expected":["analyze_discovery_readiness"],"kind":"product"},{"id":"m12","text":"We found three problems: no structured data, wrong opening hours on our site, and no FAQ page. Put them in priority order. We have two hours this week and want to be mentioned for emergency plumbing.","expected":["build_visibility_action_plan"],"kind":"product"},{"id":"m13","text":"What keywords should a plumber in Fresno go after?","expected":["keyword_research"],"kind":"control","note":"A decoy is the right tool here."},{"id":"m14","text":"What is generative engine optimization?","expected":["none"],"kind":"none"},{"id":"m15","text":"Write me a limerick about plumbers.","expected":["none"],"kind":"none"}],"decoyTools":[{"name":"seo_audit_page","description":"Crawl one web page and report on its title, headings, meta description, links, page speed and structured data. Returns a list of issues ranked by severity.","inputSchema":{"type":"object","properties":{"url":{"type":"string","format":"uri"}},"additionalProperties":false,"required":["url"]},"synthetic":true},{"name":"keyword_research","description":"Suggest search keywords from a seed phrase with monthly search volume and difficulty for a country.","inputSchema":{"type":"object","properties":{"seed":{"type":"string"},"country":{"type":"string","minLength":2,"maxLength":2},"limit":{"type":"integer","minimum":1,"maximum":100}},"additionalProperties":false,"required":["seed"]},"synthetic":true},{"name":"check_search_ranking","description":"Check where a domain ranks in Google results for a keyword in a given location.","inputSchema":{"type":"object","properties":{"keyword":{"type":"string"},"domain":{"type":"string"},"location":{"type":"string"}},"additionalProperties":false,"required":["keyword","domain"]},"synthetic":true},{"name":"fetch_webpage","description":"Fetch a web page by URL and return its readable text and links.","inputSchema":{"type":"object","properties":{"url":{"type":"string","format":"uri"},"max_chars":{"type":"integer","minimum":500,"maximum":50000}},"additionalProperties":false,"required":["url"]},"synthetic":true},{"name":"get_web_traffic","description":"Return website visits for a domain from the connected analytics account by day and by traffic source over a chosen period.","inputSchema":{"type":"object","properties":{"domain":{"type":"string"},"days":{"type":"integer","minimum":1,"maximum":365},"group_by":{"type":"string","enum":["day","source","landing_page"]}},"additionalProperties":false,"required":["domain"]},"synthetic":true},{"name":"generate_schema_markup","description":"Generate JSON-LD structured data for a page from a type such as LocalBusiness, Product or FAQ and a set of fields. Returns a script block ready to paste.","inputSchema":{"type":"object","properties":{"type":{"type":"string","enum":["LocalBusiness","Product","FAQPage","Article","Organization"]},"fields":{"type":"object"}},"additionalProperties":false,"required":["type","fields"]},"synthetic":true},{"name":"social_listening_mentions","description":"Find recent public posts and articles that mention a brand name on social networks, forums and news sites, with sentiment.","inputSchema":{"type":"object","properties":{"brand":{"type":"string"},"sources":{"type":"array","items":{"type":"string","enum":["x","reddit","news","forums"]}},"days":{"type":"integer","minimum":1,"maximum":90}},"additionalProperties":false,"required":["brand"]},"synthetic":true},{"name":"ask_llm","description":"Send one prompt to a chosen language model and return its answer. Use it to see how a model responds to a question.","inputSchema":{"type":"object","properties":{"model":{"type":"string"},"prompt":{"type":"string","maxLength":4000}},"additionalProperties":false,"required":["model","prompt"]},"synthetic":true},{"name":"get_business_reviews","description":"Fetch recent customer reviews for a business from a public review site given its name and city. Returns rating, text and date.","inputSchema":{"type":"object","properties":{"business_name":{"type":"string"},"city":{"type":"string"},"limit":{"type":"integer","minimum":1,"maximum":50}},"additionalProperties":false,"required":["business_name","city"]},"synthetic":true},{"name":"write_blog_post","description":"Write a blog post draft on a topic for a target audience and length. Returns Markdown.","inputSchema":{"type":"object","properties":{"topic":{"type":"string"},"audience":{"type":"string"},"words":{"type":"integer","minimum":200,"maximum":2500}},"additionalProperties":false,"required":["topic"]},"synthetic":true}]}}}}