{ "suite_info": { "suite_name": "WhatsApp Business Flow Master Agent – QA Conversation Suite", "part": 3, "part_description": "Categories F (Conflict/Ambiguous/Negative), G (Adversarial), H (Regression) + FAILURE_DIAGNOSIS_NOTES + EXECUTION_GUIDE", "categories_in_this_part": ["F", "G", "H"], "conversations_in_this_part": 45, "total_conversations_all_parts": 120, "created_for_agent": "WhatsApp Business Flow Master Agent", "child_agents": [ { "alias": "Onboarding Helper", "priority": 95 }, { "alias": "Template Helper", "priority": 90 }, { "alias": "Campaign Helper", "priority": 88 }, { "alias": "Chatbot Helper", "priority": 85 } ] }, "categories": { "F": { "id": "F", "name": "CONFLICT_AMBIGUOUS_NEGATIVE", "description": "Conversations where the user input is ambiguous, crosses domain boundaries, or could reasonably match multiple children. Master must resolve using active-child context, priority rules, and conflict-resolution routing.", "conversations": [ { "conversation_id": "F-001", "title": "Word 'template' mentioned inside active Campaign child context", "tags": ["conflict-resolution", "P80-rule", "campaign-active"], "previous_bad_behavior": "Master incorrectly switches to Template Helper mid-campaign because 'template' keyword fires Template Helper rule", "correct_behavior": "P=80 conflict rule: 'template' inside active Campaign child → stay in Campaign (template selection is part of campaign setup)", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to create a new campaign", "expected_intent": "create_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Campaign Helper to create your campaign.", "validation_note": "Master routes campaign creation correctly on first turn." }, { "turn_number": 2, "speaker": "BOT", "text": "Sure! What is the name for your campaign?", "expected_intent": null, "expected_target_agent": "Campaign Helper", "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Campaign Helper is asking for campaign name — slot filling in progress." }, { "turn_number": 3, "speaker": "USER", "text": "Which template should I use for this?", "expected_intent": "ask_template_selection", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Campaign Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "CRITICAL: 'template' keyword must NOT trigger a switch to Template Helper. P=80 conflict rule keeps conversation in Campaign Helper. Master must recognize this is template SELECTION within campaign context, not template CREATION." }, { "turn_number": 4, "speaker": "BOT", "text": "You can select from your approved templates in the campaign setup. Do you have an approved template ready?", "expected_intent": null, "expected_target_agent": "Campaign Helper", "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Campaign Helper correctly handles template selection question without switching agents." } ] }, { "conversation_id": "F-002", "title": "Word 'keyword' mentioned inside active Chatbot child context", "tags": ["conflict-resolution", "P78-rule", "chatbot-active"], "previous_bad_behavior": "Master switches to Onboarding or different bot because 'keyword' matched a generic rule", "correct_behavior": "P=78 rule: 'keyword' inside active Chatbot child → stay in Chatbot (keyword trigger is a chatbot concept)", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "Help me build a chatbot flow", "expected_intent": "build_chatbot_flow", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_CHATBOT_HELPER", "expected_state_update": { "active_child_agent": "Chatbot Helper", "child_agent_status_map.Chatbot Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Chatbot Helper to build your bot flow.", "validation_note": "Correct initial routing to Chatbot Helper." }, { "turn_number": 2, "speaker": "USER", "text": "How do I set a keyword trigger for welcome message?", "expected_intent": "setup_keyword_trigger", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Chatbot Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "CRITICAL: 'keyword' must NOT cause agent switch. P=78 conflict rule keeps Chatbot Helper active. Keyword trigger is a chatbot feature." }, { "turn_number": 3, "speaker": "USER", "text": "And what keywords should I avoid?", "expected_intent": "ask_keyword_restrictions", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Chatbot Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Follow-up keyword question must stay in Chatbot Helper. No domain switch." } ] }, { "conversation_id": "F-003", "title": "Word 'manual' is ambiguous — manual onboarding setup vs manual chatbot trigger", "tags": ["ambiguous-word", "clarification-needed"], "previous_bad_behavior": "Master routes 'manual' blindly to Onboarding Helper without checking for active child context", "correct_behavior": "If no active child: ASK_MINIMAL_CLARIFICATION. If Chatbot active: treat as chatbot manual trigger.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "manual", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Single word 'manual' with no active child context must trigger minimal clarification. Do NOT assume Onboarding." }, { "turn_number": 2, "speaker": "BOT", "text": "Could you clarify — are you referring to manual WhatsApp Business setup, or manual trigger for a chatbot?", "expected_intent": null, "expected_target_agent": null, "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Master asks a minimal, focused clarification question with 2 options." }, { "turn_number": 3, "speaker": "USER", "text": "manual chatbot trigger", "expected_intent": "setup_manual_trigger", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_CHATBOT_HELPER", "expected_state_update": { "active_child_agent": "Chatbot Helper", "child_agent_status_map.Chatbot Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Chatbot Helper for manual trigger setup.", "validation_note": "After clarification, routes correctly to Chatbot Helper." } ] }, { "conversation_id": "F-004", "title": "'help' request during active child session", "tags": ["generic-request", "active-child-continuation"], "previous_bad_behavior": "Master treats 'help' as new session and asks for clarification, losing active child context", "correct_behavior": "If active child exists: forward 'help' to that child (P=85 active-child continuation rule)", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I need help setting up my WhatsApp account", "expected_intent": "start_onboarding", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper", "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Onboarding Helper.", "validation_note": "Initial routing correct." }, { "turn_number": 2, "speaker": "USER", "text": "help", "expected_intent": "generic_help", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Onboarding Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "CRITICAL: 'help' during active Onboarding session must forward to Onboarding Helper, not trigger clarification. Active child context wins." } ] }, { "conversation_id": "F-005", "title": "'not working' with no active child — needs clarification", "tags": ["ambiguous-complaint", "no-active-child", "clarification"], "previous_bad_behavior": "Master routes 'not working' to Onboarding Helper by default", "correct_behavior": "No active child + vague complaint → ASK_MINIMAL_CLARIFICATION specifying which product area", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "not working", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Vague complaint with no active child. Master must NOT guess. Ask what product area is not working." }, { "turn_number": 2, "speaker": "BOT", "text": "I'd like to help! What exactly is not working — your WhatsApp account setup, message templates, campaign, or chatbot?", "expected_intent": null, "expected_target_agent": null, "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Minimal clarification question lists all 4 domains concisely." }, { "turn_number": 3, "speaker": "USER", "text": "my campaign is not sending messages", "expected_intent": "troubleshoot_campaign_delivery", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Campaign Helper to troubleshoot delivery issues.", "validation_note": "After clarification, correct routing to Campaign Helper." } ] }, { "conversation_id": "F-006", "title": "'setup issue' — could be Onboarding or Chatbot setup", "tags": ["ambiguous-complaint", "clarification-needed"], "previous_bad_behavior": "Master defaults 'setup issue' to Onboarding without checking which setup", "correct_behavior": "Ask minimal clarification: WhatsApp account setup vs chatbot flow setup", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I have a setup issue", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "'Setup issue' is ambiguous. Could be onboarding setup, chatbot setup, or account setup. Must clarify." }, { "turn_number": 2, "speaker": "USER", "text": "chatbot flow setup", "expected_intent": "setup_chatbot_flow", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_CHATBOT_HELPER", "expected_state_update": { "active_child_agent": "Chatbot Helper", "child_agent_status_map.Chatbot Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Chatbot Helper for flow setup.", "validation_note": "After clarification, correctly routes to Chatbot Helper." } ] }, { "conversation_id": "F-007", "title": "Incomplete request 'I want to create...' trailing off", "tags": ["incomplete-input", "clarification-needed"], "previous_bad_behavior": "Master forces a route guess instead of waiting for complete input", "correct_behavior": "Incomplete sentence detected → ASK_MINIMAL_CLARIFICATION to complete the request", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to create", "expected_intent": "INCOMPLETE", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Incomplete utterance. Master must ask what they want to create rather than guessing." }, { "turn_number": 2, "speaker": "BOT", "text": "Create what? A message template, a campaign, or a chatbot flow?", "expected_intent": null, "expected_target_agent": null, "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Minimal clarification with 3 specific options." }, { "turn_number": 3, "speaker": "USER", "text": "a message template", "expected_intent": "create_template", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing you to Template Helper to create your message template.", "validation_note": "Correct routing after clarification." } ] }, { "conversation_id": "F-008", "title": "User contradicts — claims onboarding done but then asks basic onboarding question", "tags": ["contradiction", "dependency-state", "context-memory"], "previous_bad_behavior": "Master accepts claim 'onboarding done' and routes Campaign request, then gets confused when user asks onboarding question", "correct_behavior": "Track dependency_status. When user asks onboarding question despite claiming done, route to Onboarding Helper without error — user may need a re-check.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "My onboarding is already done, I want to create a campaign", "expected_intent": "create_campaign_with_onboarding_claim", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "dependency_status.onboarding": "USER_CLAIMED_COMPLETE", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper. Note: dependency marked as user-claimed-complete.", "validation_note": "Master accepts user claim and routes to Campaign Helper. Records dependency as USER_CLAIMED_COMPLETE (not verified)." }, { "turn_number": 2, "speaker": "USER", "text": "Actually, what is an App ID and where do I find it?", "expected_intent": "ask_app_id", "expected_target_agent": "Onboarding Helper", "expected_master_action": "PRESERVE_CONTEXT_AND_SWITCH", "expected_state_update": { "active_child_agent": "Onboarding Helper", "child_agent_status_map.Campaign Helper": "PAUSED", "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": { "paused_task": "campaign_creation" }, "expected_handoff": "Switching to Onboarding Helper to answer your App ID question. Your campaign task is saved.", "validation_note": "Onboarding question overrides current Campaign context. Master switches gracefully without error. App ID is an onboarding concept — routes to Onboarding Helper." } ] }, { "conversation_id": "F-009", "title": "'publish' ambiguous — chatbot publish vs template submission", "tags": ["ambiguous-word", "publish", "conflict-resolution"], "previous_bad_behavior": "Master routes 'publish' to Template Helper (for template approval submission) when Chatbot is active", "correct_behavior": "P=75 conflict rule: 'publish' during active Chatbot child → stay in Chatbot (publishing chatbot flow is a chatbot operation)", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to set up my chatbot", "expected_intent": "setup_chatbot", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_CHATBOT_HELPER", "expected_state_update": { "active_child_agent": "Chatbot Helper", "child_agent_status_map.Chatbot Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Chatbot Helper.", "validation_note": "Initial routing correct." }, { "turn_number": 2, "speaker": "USER", "text": "How do I publish it?", "expected_intent": "publish_chatbot", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Chatbot Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "CRITICAL: P=75 rule — 'publish' during active Chatbot → stay in Chatbot. Must NOT switch to Template Helper." } ] }, { "conversation_id": "F-010", "title": "'status' ambiguous — campaign status vs template status vs onboarding status", "tags": ["ambiguous-word", "status", "active-child-wins"], "previous_bad_behavior": "Master ignores active child and asks clarification every time 'status' is mentioned", "correct_behavior": "If active child present: forward 'status' query to active child. If no active child: clarify.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I'm working on a campaign", "expected_intent": "start_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper.", "validation_note": "Initial routing correct." }, { "turn_number": 2, "speaker": "USER", "text": "What is the status?", "expected_intent": "check_campaign_status", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Campaign Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Active child wins. 'status' forwarded to Campaign Helper as campaign status query. No clarification needed when active child is clear." } ] }, { "conversation_id": "F-011", "title": "Template mention inside campaign — asking about template content mid-campaign", "tags": ["conflict-resolution", "P80-rule", "template-in-campaign"], "previous_bad_behavior": "Master switches to Template Helper when user says 'what does this template say' during campaign setup", "correct_behavior": "Template content question during campaign setup = campaign child question. P=80 keeps Campaign active.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to launch a broadcast campaign", "expected_intent": "create_broadcast_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper for broadcast campaign setup.", "validation_note": "Correct initial routing." }, { "turn_number": 2, "speaker": "USER", "text": "I see a template called 'promo_offer' — what does it contain?", "expected_intent": "ask_template_content_in_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Campaign Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "CRITICAL: Template content inquiry during campaign setup stays with Campaign Helper. P=80 conflict rule enforced. Do NOT switch to Template Helper." } ] }, { "conversation_id": "F-012", "title": "User says 'go back' — expects return to master", "tags": ["navigation", "return-to-master", "explicit-switch"], "previous_bad_behavior": "Master ignores 'go back' and continues with current child", "correct_behavior": "Explicit navigation 'go back' → RETURN_CONTROL_TO_MASTER, clear active child, offer master-level options", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I need help with templates", "expected_intent": "start_template", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper.", "validation_note": "Initial routing correct." }, { "turn_number": 2, "speaker": "USER", "text": "go back", "expected_intent": "navigate_back_to_master", "expected_target_agent": null, "expected_master_action": "RETURN_CONTROL_TO_MASTER", "expected_state_update": { "active_child_agent": null, "child_agent_status_map.Template Helper": "PAUSED" }, "expected_queue_update": { "saved_context": "Template Helper session paused" }, "expected_handoff": "Returning to main menu. What else can I help you with?", "validation_note": "Explicit navigation command triggers RETURN_CONTROL_TO_MASTER. Active child cleared. Previous session context preserved for potential resume." } ] }, { "conversation_id": "F-013", "title": "Mixed domain message — template AND campaign in same utterance", "tags": ["multi-intent", "priority-order", "queue-secondary"], "previous_bad_behavior": "Master picks only one domain and ignores the other", "correct_behavior": "Primary task identified (template creation prerequisite); campaign queued as secondary", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to create a promo template and also set up a campaign for it", "expected_intent": "create_template_then_campaign", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": { "queued": "Campaign Helper — create campaign using promo template" }, "expected_handoff": "Starting with template creation first. Campaign setup will follow after template is approved.", "validation_note": "Both domains identified. Template is prerequisite for campaign — Template Helper first, Campaign queued. Master communicates the plan to user." } ] }, { "conversation_id": "F-014", "title": "Negative response to clarification question — user says 'no' to all options", "tags": ["negative-response", "clarification-dead-end"], "previous_bad_behavior": "Master gets stuck in clarification loop or crashes on 'no' response", "correct_behavior": "After 'no' to clarification, master acknowledges and invites free-form description", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I have an issue", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Ambiguous complaint triggers minimal clarification." }, { "turn_number": 2, "speaker": "BOT", "text": "Is this about account setup, templates, campaigns, or chatbot?", "expected_intent": null, "expected_target_agent": null, "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Standard clarification question." }, { "turn_number": 3, "speaker": "USER", "text": "none of those", "expected_intent": "NEGATIVE_CLARIFICATION", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "User rejects all options. Master must NOT crash or loop. Invite free description: 'Could you describe what you are trying to do?'" } ] }, { "conversation_id": "F-015", "title": "User says 'I changed my mind' mid-flow — abandon current task", "tags": ["task-abandonment", "context-reset"], "previous_bad_behavior": "Master continues current task despite explicit abandonment signal", "correct_behavior": "Explicit abandonment → RETURN_CONTROL_TO_MASTER, clear active child, offer fresh start", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "Let me create a template", "expected_intent": "create_template", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper.", "validation_note": "Initial routing." }, { "turn_number": 2, "speaker": "USER", "text": "Actually I changed my mind, forget it", "expected_intent": "abandon_task", "expected_target_agent": null, "expected_master_action": "RETURN_CONTROL_TO_MASTER", "expected_state_update": { "active_child_agent": null, "child_agent_status_map.Template Helper": "NOT_STARTED" }, "expected_queue_update": null, "expected_handoff": "No problem! What would you like to do instead?", "validation_note": "Task abandonment clears active child and resets Template Helper status. Master offers fresh options." } ] } ] }, "G": { "id": "G", "name": "ADVERSARIAL", "description": "Inputs designed to break routing logic — pronouns without referents, injected URLs/numbers, single words, intentional dependency bypasses, contradictory commands, and off-topic injections.", "conversations": [ { "conversation_id": "G-001", "title": "Implied campaign request — 'I want to send offers on WhatsApp' — but onboarding not done", "tags": ["adversarial", "dependency-check", "implied-campaign"], "previous_bad_behavior": "Master routes to Campaign Helper without checking onboarding dependency", "correct_behavior": "Implied campaign detected → check dependency_status → onboarding NOT done → RESOLVE_DEPENDENCY_FIRST → route to Onboarding Helper", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to send offers on WhatsApp", "expected_intent": "send_broadcast_campaign", "expected_target_agent": "Onboarding Helper", "expected_master_action": "RESOLVE_DEPENDENCY_FIRST", "expected_state_update": { "active_child_agent": "Onboarding Helper", "blocked_task_list": ["Campaign Helper — send offers"], "dependency_status.onboarding": "REQUIRED" }, "expected_queue_update": { "queued": "Campaign Helper — send offers after onboarding" }, "expected_handoff": "To send offers via WhatsApp, you first need a WhatsApp Business account. Let me help you get set up first.", "validation_note": "CRITICAL: 'send offers' implies campaign. But dependency check blocks it. Master must route to Onboarding Helper and queue campaign task. Must NOT go directly to Campaign Helper." } ] }, { "conversation_id": "G-002", "title": "'Use this template in my campaign' — cross-domain reference with pronoun", "tags": ["adversarial", "cross-domain", "pronoun-reference"], "previous_bad_behavior": "Master gets confused by pronoun 'this' and fails to route either task", "correct_behavior": "Detects campaign intent as primary; 'this template' = existing template reference in campaign context. Route to Campaign Helper with template reference context.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "Use this template in my campaign", "expected_intent": "assign_template_to_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper to assign the template to your campaign.", "validation_note": "Primary intent is campaign configuration. Template is an object in the campaign context, not a new template creation task. Campaign Helper handles template assignment." } ] }, { "conversation_id": "G-003", "title": "Dual simultaneous complaint — 'bot not replying AND I also need onboarding'", "tags": ["adversarial", "dual-task", "dependency", "multi-intent"], "previous_bad_behavior": "Master handles only one of the two tasks, ignores the other", "correct_behavior": "Two explicit tasks identified: onboarding (dependency) + chatbot troubleshoot. Onboarding is prerequisite → ROUTE_TO_ONBOARDING_HELPER first, queue chatbot task.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "My bot is not replying but I also need onboarding", "expected_intent": "dual_task_onboarding_and_chatbot_issue", "expected_target_agent": "Onboarding Helper", "expected_master_action": "RESOLVE_DEPENDENCY_FIRST", "expected_state_update": { "active_child_agent": "Onboarding Helper", "child_agent_status_map.Onboarding Helper": "ACTIVE", "blocked_task_list": ["Chatbot Helper — bot not replying"] }, "expected_queue_update": { "queued": "Chatbot Helper — troubleshoot bot not replying" }, "expected_handoff": "I see two things to handle: onboarding setup and chatbot issue. Let's complete onboarding first since it's a dependency. Chatbot troubleshoot is saved for next.", "validation_note": "Explicit dual task. Onboarding is prerequisite for chatbot. Master must communicate both tasks are acknowledged, prioritize onboarding, and queue chatbot explicitly." } ] }, { "conversation_id": "G-004", "title": "Single word 'manual' with NO session context", "tags": ["adversarial", "single-word", "no-context"], "previous_bad_behavior": "Master crashes, throws 500 error, or blindly routes to Onboarding Helper", "correct_behavior": "Single word with no session context → ASK_MINIMAL_CLARIFICATION. No crash, no blind guess.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "manual", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Single ambiguous word with zero session context. Master must handle gracefully with minimal clarification question." } ] }, { "conversation_id": "G-005", "title": "Bare 'yes' with no preceding question", "tags": ["adversarial", "affirmative-no-context", "no-session"], "previous_bad_behavior": "Master treats 'yes' as confirmation of unknown action, routes to wrong child", "correct_behavior": "Bare 'yes' with no session state → fallback/clarification. Cannot confirm something unknown.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "yes", "expected_intent": "AFFIRMATIVE_NO_CONTEXT", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Bare affirmative with no pending question in session history. Master must not assume what is being confirmed. Ask what user wants to do." } ] }, { "conversation_id": "G-006", "title": "'create it first' — dangling pronoun reference with no prior context", "tags": ["adversarial", "pronoun-no-referent", "clarification"], "previous_bad_behavior": "Master attempts to route to some child based on 'create' keyword alone, ignoring undefined 'it'", "correct_behavior": "Pronoun 'it' with no referent → ASK_MINIMAL_CLARIFICATION: create what?", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "create it first", "expected_intent": "AMBIGUOUS_PRONOUN", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Pronoun 'it' without referent in session history. Master must ask: 'Create what first — a template, campaign, or chatbot?'" }, { "turn_number": 2, "speaker": "USER", "text": "the message template", "expected_intent": "create_template", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper to create the message template.", "validation_note": "After clarification, correct routing to Template Helper." } ] }, { "conversation_id": "G-007", "title": "URL injected as user input — 'https://example.com'", "tags": ["adversarial", "url-injection", "safe-handling"], "previous_bad_behavior": "Master attempts to fetch or route based on the URL, or crashes", "correct_behavior": "Bare URL with no session context → treat as unrecognized input. Ask what user needs help with. Do NOT follow URL.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "https://example.com", "expected_intent": "UNRECOGNIZED", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "URL injection. Master must NOT fetch the URL or fail. Treat as unrecognized input and ask what the user needs help with. Graceful degradation required." } ] }, { "conversation_id": "G-008", "title": "Phone number injected as user input — '123456789'", "tags": ["adversarial", "number-injection", "safe-handling"], "previous_bad_behavior": "Master matches number to Phone Number ID slot and routes to Onboarding mid-conversation", "correct_behavior": "Standalone number with no active slot-filling context → treat as unrecognized, ask clarification", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "123456789", "expected_intent": "UNRECOGNIZED", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Bare number with no active child and no active slot-filling context. Must NOT be treated as phone number ID or phone number parameter. Ask what user needs." } ] }, { "conversation_id": "G-009", "title": "'not this go back' — navigation plus negation combo", "tags": ["adversarial", "navigation", "negation-combo"], "previous_bad_behavior": "Master ignores 'go back' and tries to match 'not this' as an intent", "correct_behavior": "Navigation command 'go back' in combo with negation → RETURN_CONTROL_TO_MASTER. Acknowledge that user rejects current path.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I need help with campaign analytics", "expected_intent": "campaign_analytics", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper for analytics.", "validation_note": "Initial routing." }, { "turn_number": 2, "speaker": "USER", "text": "not this go back", "expected_intent": "navigate_back_reject", "expected_target_agent": null, "expected_master_action": "RETURN_CONTROL_TO_MASTER", "expected_state_update": { "active_child_agent": null, "child_agent_status_map.Campaign Helper": "PAUSED" }, "expected_queue_update": null, "expected_handoff": "Understood, going back. What would you like help with?", "validation_note": "Composite navigation+negation must trigger RETURN_CONTROL_TO_MASTER cleanly. Active child paused, not abandoned." } ] }, { "conversation_id": "G-010", "title": "'no template but make campaign' — explicit dependency bypass attempt", "tags": ["adversarial", "dependency-bypass", "explicit-block"], "previous_bad_behavior": "Master accepts the bypass and routes to Campaign Helper without template", "correct_behavior": "WhatsApp campaigns require approved templates. Master must block campaign creation and explain the dependency.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I don't want to create a template, just make the campaign directly", "expected_intent": "bypass_template_dependency", "expected_target_agent": "Template Helper", "expected_master_action": "RESOLVE_DEPENDENCY_FIRST", "expected_state_update": { "active_child_agent": "Template Helper", "blocked_task_list": ["Campaign Helper — direct campaign without template"], "dependency_status.template": "REQUIRED" }, "expected_queue_update": { "queued": "Campaign Helper — after template approved" }, "expected_handoff": "WhatsApp campaigns require an approved message template. Let me help you create one first — it's quick and required by WhatsApp's policy.", "validation_note": "CRITICAL: User explicitly tries to bypass template dependency. Master must NOT allow Campaign Helper routing. Must explain requirement and route to Template Helper. Dependency rule P=110 enforced." } ] }, { "conversation_id": "G-011", "title": "'I already finished setup now what' — ambiguous completion claim with no verified state", "tags": ["adversarial", "ambiguous-completion", "next-step-guidance"], "previous_bad_behavior": "Master blindly accepts all dependencies as complete and presents all options", "correct_behavior": "Acknowledge completion claim, ask which setup they finished to confirm next step", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I already finished setup now what", "expected_intent": "post_setup_next_steps", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Ambiguous completion claim. 'Setup' could mean onboarding, chatbot setup, or template setup. Master must ask minimal clarification: 'Which setup did you finish — WhatsApp account, template, campaign, or chatbot?'" }, { "turn_number": 2, "speaker": "USER", "text": "WhatsApp account setup, onboarding is done", "expected_intent": "onboarding_complete_next_steps", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": { "dependency_status.onboarding": "USER_CLAIMED_COMPLETE" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "After clarification, master knows onboarding is claimed complete. Now offer next steps: template creation or campaign — ask which they'd like to do next." } ] }, { "conversation_id": "G-012", "title": "Bare 'help' with no session context", "tags": ["adversarial", "bare-help", "master-level-fallback"], "previous_bad_behavior": "Master routes to Onboarding Helper as default for 'help'", "correct_behavior": "Bare 'help' with no session context → master-level welcome/options menu, not a specific child route", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "help", "expected_intent": "request_help", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Bare 'help' with fresh session. Master presents a brief overview: 'I can help with: 1) WhatsApp account setup 2) Message templates 3) Campaigns 4) Chatbot. What do you need?' Do NOT route to any specific child." } ] }, { "conversation_id": "G-013", "title": "Bare 'not working' with NO session context — must not crash", "tags": ["adversarial", "vague-complaint", "no-session"], "previous_bad_behavior": "Master throws error or defaults to Onboarding Helper for vague complaint", "correct_behavior": "Vague complaint with no session → graceful clarification. Never crash. Never default-route.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "not working", "expected_intent": "AMBIGUOUS_COMPLAINT", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "System stability test. 'Not working' must be handled without crash, 500 error, or blind routing. Must produce a minimal clarification question." } ] }, { "conversation_id": "G-014", "title": "Emoji-only input — no text content", "tags": ["adversarial", "emoji-input", "graceful-degradation"], "previous_bad_behavior": "Master crashes or throws uncaught exception on emoji-only input", "correct_behavior": "Non-text or emoji input → graceful fallback. Ask user to describe their question in words.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "👋🤔❓", "expected_intent": "UNRECOGNIZED", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Emoji-only input must not crash master agent. Must respond with friendly prompt to describe question in words. Test for input sanitization." } ] }, { "conversation_id": "G-015", "title": "Cross-domain confusion attack — repeatedly switching domains to break sticky agent", "tags": ["adversarial", "rapid-switching", "sticky-agent-robustness"], "previous_bad_behavior": "Rapid domain switching causes master to lose session state or carry wrong active_child", "correct_behavior": "Each explicit domain switch is valid and recorded. State remains consistent. No ghost active-child from previous turn.", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to set up my WhatsApp", "expected_intent": "start_onboarding", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper" }, "expected_queue_update": null, "expected_handoff": "Routing to Onboarding Helper.", "validation_note": "Initial state established." }, { "turn_number": 2, "speaker": "USER", "text": "Actually create a template", "expected_intent": "create_template", "expected_target_agent": "Template Helper", "expected_master_action": "PRESERVE_CONTEXT_AND_SWITCH", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Onboarding Helper": "PAUSED" }, "expected_queue_update": null, "expected_handoff": "Switching to Template Helper. Onboarding saved.", "validation_note": "Explicit switch to Template Helper. Onboarding paused cleanly." }, { "turn_number": 3, "speaker": "USER", "text": "No, campaign instead", "expected_intent": "switch_to_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "PRESERVE_CONTEXT_AND_SWITCH", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Template Helper": "PAUSED" }, "expected_queue_update": null, "expected_handoff": "Switching to Campaign Helper. Template session saved.", "validation_note": "Second rapid switch. Must correctly pause Template Helper and activate Campaign Helper. No ghost states." }, { "turn_number": 4, "speaker": "USER", "text": "chatbot", "expected_intent": "switch_to_chatbot", "expected_target_agent": "Chatbot Helper", "expected_master_action": "PRESERVE_CONTEXT_AND_SWITCH", "expected_state_update": { "active_child_agent": "Chatbot Helper", "child_agent_status_map.Campaign Helper": "PAUSED" }, "expected_queue_update": null, "expected_handoff": "Switching to Chatbot Helper. Campaign session saved.", "validation_note": "Third rapid switch. State must be: Onboarding=PAUSED, Template=PAUSED, Campaign=PAUSED, Chatbot=ACTIVE. No corruption." } ] } ] }, "H": { "id": "H", "name": "REGRESSION", "description": "Explicit regression tests for previously observed or anticipated failures. Each test documents: the old bad behavior, the fixed correct behavior, and the exact test scenario. These MUST pass after each deployment.", "conversations": [ { "conversation_id": "H-001", "title": "REGRESSION: Multi-slot capture — date + city in same message", "tags": ["regression", "multi-slot", "entity-extraction"], "previous_bad_behavior": "When user provides date AND city in same utterance, only one slot was captured; second slot was lost or not extracted", "correct_behavior": "Both slots extracted in same turn. No reprompt for already-provided slot.", "correct_master_route": "Onboarding Helper → slot-filling pipeline captures both parameters in step 8", "correct_child_continuity": "Onboarding Helper does NOT ask for slot that was already provided in turn 1", "correct_state_memory": "Both slots recorded in session context after turn 1", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to schedule my onboarding call on 25 March in Mumbai", "expected_intent": "schedule_onboarding_call", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper", "extracted_params": { "date": "25 March", "city": "Mumbai" } }, "expected_queue_update": null, "expected_handoff": "Routing to Onboarding Helper with date and city captured.", "validation_note": "REGRESSION TEST: Both 'date=25 March' and 'city=Mumbai' must be extracted in single turn. entity extraction must capture BOTH sys.date and sys.city parameters simultaneously." }, { "turn_number": 2, "speaker": "BOT", "text": "Your onboarding call is scheduled for 25 March in Mumbai. What time works for you?", "expected_intent": null, "expected_target_agent": "Onboarding Helper", "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Bot must NOT re-ask for date or city. Both were captured. Only missing slot (time) is asked." } ] }, { "conversation_id": "H-002", "title": "REGRESSION: Phone Number ID '120364abc' must NOT be classified as a phone number", "tags": ["regression", "phone-number-id", "entity-classification"], "previous_bad_behavior": "Phone Number ID (long alphanumeric or numeric ID from Meta dashboard) was being extracted as sys.phone, causing wrong slot fill", "correct_behavior": "Phone Number ID slot must be extracted as a distinct entity type, not as sys.phone. The slot name in the intent definition is 'phone_number_id', not 'phone_number'.", "correct_master_route": "Onboarding Helper — phone_number_id slot correctly filled", "correct_child_continuity": "Onboarding Helper does NOT treat this as a contact phone number", "correct_state_memory": "Session records phone_number_id = '1203641234567', not phone = '1203641234567'", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "My Phone Number ID is 1203641234567", "expected_intent": "provide_phone_number_id", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper", "extracted_params": { "phone_number_id": "1203641234567" } }, "expected_queue_update": null, "expected_handoff": "Captured Phone Number ID for onboarding.", "validation_note": "REGRESSION TEST: '1203641234567' must be extracted into slot 'phone_number_id', NOT into 'sys.phone'. These are different entities. Phone Number ID is a Meta API identifier, not a contact number." } ] }, { "conversation_id": "H-003", "title": "REGRESSION: 'template' mention in active campaign context must NOT switch to Template Helper", "tags": ["regression", "conflict-P80", "wrong-switch-prevention"], "previous_bad_behavior": "User saying 'which template should I pick' during campaign setup caused master to switch to Template Helper — destroying campaign slot-fill context", "correct_behavior": "P=80 conflict rule enforced: 'template' in active Campaign child context → stay in Campaign Helper", "correct_master_route": "Campaign Helper stays active", "correct_child_continuity": "Campaign slot filling continues; template question answered by Campaign Helper", "correct_state_memory": "active_child_agent remains 'Campaign Helper' throughout", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "Create a new campaign for Diwali offers", "expected_intent": "create_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper.", "validation_note": "Campaign context established." }, { "turn_number": 2, "speaker": "BOT", "text": "Great! What message template do you want to use for this campaign?", "expected_intent": null, "expected_target_agent": "Campaign Helper", "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Campaign Helper is asking for template selection." }, { "turn_number": 3, "speaker": "USER", "text": "I am not sure which template to use", "expected_intent": "unsure_about_template_selection", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Campaign Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: 'template' in this response must NOT cause switch to Template Helper. P=80 conflict rule must fire. Campaign Helper handles template guidance within campaign context." } ] }, { "conversation_id": "H-004", "title": "REGRESSION: Short contextual reply with NO active-agent awareness", "tags": ["regression", "sticky-agent", "short-reply"], "previous_bad_behavior": "Short reply like 'Evening' or '5pm' during slot-filling caused master to lose sticky agent — either re-routing or asking clarification", "correct_behavior": "Sticky agent bias: short replies during active child session → forward to active child as slot response. Do NOT re-route.", "correct_master_route": "Onboarding Helper receives 'Evening' as time slot fill", "correct_child_continuity": "Slot filled successfully; next slot prompted", "correct_state_memory": "active_child_agent = 'Onboarding Helper' unchanged", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to book an onboarding call", "expected_intent": "book_onboarding_call", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper", "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Onboarding Helper.", "validation_note": "Onboarding context established." }, { "turn_number": 2, "speaker": "BOT", "text": "What time of day works for you?", "expected_intent": null, "expected_target_agent": "Onboarding Helper", "expected_master_action": null, "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Slot prompt for time." }, { "turn_number": 3, "speaker": "USER", "text": "Evening", "expected_intent": "provide_time_slot", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Onboarding Helper", "extracted_params": { "preferred_time": "Evening" } }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: Single word 'Evening' during active Onboarding slot-filling must be forwarded to Onboarding Helper as slot response. Sticky agent bias applied. Must NOT trigger clarification or re-routing." } ] }, { "conversation_id": "H-005", "title": "REGRESSION: Campaign creation attempted before onboarding — dependency block must fire", "tags": ["regression", "dependency-block", "P110-rule"], "previous_bad_behavior": "Master routed to Campaign Helper for 'create campaign' without checking onboarding dependency, leading to campaign setup for unverified WhatsApp accounts", "correct_behavior": "P=110 dependency block rule: if onboarding NOT marked complete, 'create campaign' → RESOLVE_DEPENDENCY_FIRST → Onboarding Helper", "correct_master_route": "Onboarding Helper (not Campaign Helper)", "correct_child_continuity": "Campaign task queued as pending; resumes after onboarding complete", "correct_state_memory": "dependency_status.onboarding = 'REQUIRED'; blocked_task_list contains campaign task", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to create a campaign right now", "expected_intent": "create_campaign", "expected_target_agent": "Onboarding Helper", "expected_master_action": "RESOLVE_DEPENDENCY_FIRST", "expected_state_update": { "active_child_agent": "Onboarding Helper", "blocked_task_list": ["Campaign Helper — create campaign"], "dependency_status.onboarding": "REQUIRED", "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": { "queued": "Campaign Helper — create campaign" }, "expected_handoff": "Before creating a campaign, your WhatsApp Business account needs to be set up. Let me take you through onboarding first.", "validation_note": "REGRESSION TEST: 'create campaign' with fresh session (no onboarding completed) MUST trigger RESOLVE_DEPENDENCY_FIRST. P=110 rule. Master must NOT route to Campaign Helper." } ] }, { "conversation_id": "H-006", "title": "REGRESSION: 'is me eligible to use waba?' mid-onboarding must hit ask_eligibility_mid_flow", "tags": ["regression", "intent-routing", "context-gated-intent", "known-bug-fixed"], "previous_bad_behavior": "Mid-onboarding eligibility question matched global check_eligibility at 52% confidence instead of context-gated ask_eligibility_mid_flow. User got pre-onboarding eligibility checklist instead of reassurance.", "correct_behavior": "With onboarding-active context present, ask_eligibility_mid_flow intercepts at higher confidence due to context boost. User gets reassurance that they are eligible (already in onboarding).", "correct_master_route": "Onboarding Helper with ask_eligibility_mid_flow intent", "correct_child_continuity": "Onboarding flow continues after eligibility reassurance", "correct_state_memory": "onboarding-active context lifespan reset to 20", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to start WhatsApp Business API setup", "expected_intent": "start_onboarding", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ONBOARDING_HELPER", "expected_state_update": { "active_child_agent": "Onboarding Helper", "active_context": "onboarding-active:20" }, "expected_queue_update": null, "expected_handoff": "Routing to Onboarding Helper.", "validation_note": "onboarding-active context is established with lifespan 20." }, { "turn_number": 2, "speaker": "BOT", "text": "Let's start! First, go to business.facebook.com and create a Meta Business Account...", "expected_intent": null, "expected_target_agent": "Onboarding Helper", "expected_master_action": null, "expected_state_update": { "active_context": "onboarding-active:19" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Onboarding in progress. Context decrements." }, { "turn_number": 3, "speaker": "USER", "text": "is me eligible to use waba?", "expected_intent": "ask_eligibility_mid_flow", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Onboarding Helper", "active_context": "onboarding-active:20" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST (KNOWN BUG FIXED): With onboarding-active:19 present, 'is me eligible to use waba?' MUST match ask_eligibility_mid_flow (context-gated intent), NOT check_eligibility (global intent). Context boost ensures correct routing. Response should reassure user they are already in onboarding and thus eligible." } ] }, { "conversation_id": "H-007", "title": "REGRESSION: Chatbot setup requested before onboarding — soft dependency check", "tags": ["regression", "soft-dependency", "chatbot-onboarding"], "previous_bad_behavior": "Master routed to Chatbot Helper without checking onboarding status, allowing chatbot setup for accounts not yet on WhatsApp API", "correct_behavior": "Onboarding is a soft dependency for Chatbot Helper. If onboarding not done, master informs user but can still route to Chatbot Helper with a warning. Does not hard-block like campaign.", "correct_master_route": "Chatbot Helper (after warning about onboarding dependency)", "correct_child_continuity": "Chatbot Helper proceeds with setup guidance; onboarding queued as recommended next step", "correct_state_memory": "dependency_status.onboarding = 'WARNING_ISSUED'; active_child = Chatbot Helper", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to build a chatbot", "expected_intent": "build_chatbot", "expected_target_agent": "Chatbot Helper", "expected_master_action": "ROUTE_TO_CHATBOT_HELPER", "expected_state_update": { "active_child_agent": "Chatbot Helper", "dependency_status.onboarding": "WARNING_ISSUED", "child_agent_status_map.Chatbot Helper": "ACTIVE" }, "expected_queue_update": { "recommended": "Onboarding Helper — complete WhatsApp account setup to deploy chatbot" }, "expected_handoff": "Routing to Chatbot Helper. Note: to deploy your chatbot, you'll also need a WhatsApp Business account set up. We can help with that after.", "validation_note": "REGRESSION TEST: Chatbot before onboarding = SOFT dependency. Master warns but does NOT block. Routes to Chatbot Helper with warning. Campaign before onboarding = HARD dependency (H-005) — this is different behavior." } ] }, { "conversation_id": "H-008", "title": "REGRESSION: Campaign creation after template done but onboarding NOT done — hard block still applies", "tags": ["regression", "dependency-chain", "hard-block"], "previous_bad_behavior": "Master saw template as completed and assumed onboarding was also done (false inference). Routed to Campaign Helper skipping onboarding check.", "correct_behavior": "Template completion does NOT imply onboarding completion. Each dependency is tracked independently. Campaign still blocked if onboarding not done.", "correct_master_route": "Onboarding Helper (campaign blocked until onboarding done)", "correct_child_continuity": "Campaign queued after onboarding; template status preserved", "correct_state_memory": "dependency_status.onboarding = 'REQUIRED'; dependency_status.template = 'COMPLETED'; blocked_task_list contains campaign", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I finished creating my template. Now I want to run a campaign.", "expected_intent": "create_campaign_after_template", "expected_target_agent": "Onboarding Helper", "expected_master_action": "RESOLVE_DEPENDENCY_FIRST", "expected_state_update": { "active_child_agent": "Onboarding Helper", "dependency_status.template": "COMPLETED", "dependency_status.onboarding": "REQUIRED", "blocked_task_list": ["Campaign Helper — run campaign"], "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": { "queued": "Campaign Helper — run campaign after onboarding" }, "expected_handoff": "Template is ready! But to run a campaign, you also need your WhatsApp Business account set up first. Let me help with onboarding.", "validation_note": "REGRESSION TEST: Template=DONE does NOT mean Onboarding=DONE. Dependency check must be independent for each prerequisite. Campaign still blocked until onboarding explicitly completed or claimed complete." } ] }, { "conversation_id": "H-009", "title": "REGRESSION: Gibberish/nonsense input must not crash master", "tags": ["regression", "gibberish", "stability-test"], "previous_bad_behavior": "Gibberish input caused uncaught exception in intent matcher or master routing — resulting in 500 error", "correct_behavior": "All inputs handled gracefully. Gibberish → fallback/clarification. No 500. No crash.", "correct_master_route": "No route. Graceful fallback with clarification.", "correct_child_continuity": "N/A — no child routed", "correct_state_memory": "Session state unchanged", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "xzqw123 @@@ bloop norp", "expected_intent": "UNRECOGNIZED", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: Nonsense input must be handled gracefully. HTTP 200 response expected. No 500 errors. No crashes. Fallback with polite clarification prompt." } ] }, { "conversation_id": "H-010", "title": "REGRESSION: Sticky agent over-application after task completed", "tags": ["regression", "sticky-agent", "post-completion-routing"], "previous_bad_behavior": "After Onboarding Helper marked complete, subsequent 'create template' request was still routed back to Onboarding Helper due to stale sticky agent state", "correct_behavior": "Once child task is marked COMPLETED, active_child_agent must be cleared. Next distinct domain request must route freshly, not sticky to completed child.", "correct_master_route": "Template Helper (NOT Onboarding Helper)", "correct_child_continuity": "New Template Helper session started fresh", "correct_state_memory": "child_agent_status_map.Onboarding Helper = COMPLETED; active_child_agent = Template Helper", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I finished the onboarding process", "expected_intent": "onboarding_complete", "expected_target_agent": "Onboarding Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Onboarding Helper", "child_agent_status_map.Onboarding Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Onboarding completion acknowledged." }, { "turn_number": 2, "speaker": "BOT", "text": "Great, your WhatsApp Business account is fully set up! What would you like to do next?", "expected_intent": null, "expected_target_agent": null, "expected_master_action": "RETURN_CONTROL_TO_MASTER", "expected_state_update": { "active_child_agent": null, "child_agent_status_map.Onboarding Helper": "COMPLETED", "dependency_status.onboarding": "COMPLETED" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "Onboarding child marks itself COMPLETED. active_child_agent cleared. Control returns to master." }, { "turn_number": 3, "speaker": "USER", "text": "Now create a message template", "expected_intent": "create_template", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper to create your message template.", "validation_note": "REGRESSION TEST: After Onboarding COMPLETED and active_child cleared, 'create template' must route to Template Helper — NOT back to Onboarding Helper. Sticky agent must NOT survive task completion." } ] }, { "conversation_id": "H-011", "title": "REGRESSION: Template category question mid-campaign must stay in Campaign", "tags": ["regression", "conflict-P80", "template-category-in-campaign"], "previous_bad_behavior": "User asking 'which template category should I use for this campaign?' caused switch to Template Helper (template creation flow) instead of staying in Campaign Helper (template selection)", "correct_behavior": "Template category guidance in campaign context = Campaign Helper responsibility. P=80 conflict rule keeps Campaign active.", "correct_master_route": "Campaign Helper stays active", "correct_child_continuity": "Campaign slot filling continues", "correct_state_memory": "active_child_agent = Campaign Helper throughout", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "Set up a campaign for my Eid sale", "expected_intent": "create_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper.", "validation_note": "Campaign context established." }, { "turn_number": 2, "speaker": "USER", "text": "Which template category should I use — marketing or utility?", "expected_intent": "ask_template_category_guidance", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_ACTIVE_CHILD", "expected_state_update": { "active_child_agent": "Campaign Helper" }, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: 'template category' during active campaign must NOT switch to Template Helper. P=80 rule enforced. Campaign Helper answers template category question in campaign context." } ] }, { "conversation_id": "H-012", "title": "REGRESSION: Session resume after context window expiry", "tags": ["regression", "context-expiry", "session-resume"], "previous_bad_behavior": "After context lifespan reached 0, master lost track of which child was active. Subsequent input matched wrong child or caused clarification loop.", "correct_behavior": "Context expiry does not permanently lose session state if DB session is still valid. Master re-establishes from session DB, not from context alone.", "correct_master_route": "Master re-checks session DB for active_child_agent. Resumes correctly.", "correct_child_continuity": "Correct child resumed based on DB session state", "correct_state_memory": "Session DB is source of truth, not context lifespan counter alone", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I was setting up my WhatsApp campaign earlier", "expected_intent": "resume_campaign_session", "expected_target_agent": "Campaign Helper", "expected_master_action": "RESUME_PREVIOUS_TASK", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Welcome back! Resuming your campaign setup from where you left off.", "validation_note": "REGRESSION TEST: Context lifespan may have expired between sessions, but DB session record still has active_child_agent = Campaign Helper and campaign slot state. Master reads DB, resumes correctly. Must NOT ask user to start over." } ] }, { "conversation_id": "H-013", "title": "REGRESSION: 'I want to add a button' — ambiguous between chatbot button and template button", "tags": ["regression", "ambiguous-button", "clarification-needed"], "previous_bad_behavior": "Master defaulted to Chatbot Helper for 'add a button' because chatbot was mentioned in recent session history", "correct_behavior": "Without active child context, 'button' is ambiguous between chatbot flow button and template quick-reply/URL button. Must clarify.", "correct_master_route": "ASK_MINIMAL_CLARIFICATION: 'Is this button for a message template or a chatbot flow?'", "correct_child_continuity": "Route to correct child AFTER clarification", "correct_state_memory": "No premature active_child set before clarification", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to add a button", "expected_intent": "AMBIGUOUS_BUTTON", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: 'button' is ambiguous. Template quick-reply/URL buttons = Template Helper. Chatbot node buttons = Chatbot Helper. Must ask minimal clarification without assuming chatbot." }, { "turn_number": 2, "speaker": "USER", "text": "for my message template, a quick reply button", "expected_intent": "add_template_quick_reply_button", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper to add a quick reply button to your template.", "validation_note": "After clarification, correct routing to Template Helper for button configuration." } ] }, { "conversation_id": "H-014", "title": "REGRESSION: 'schedule' ambiguous — campaign schedule vs chatbot trigger schedule", "tags": ["regression", "ambiguous-schedule", "clarification"], "previous_bad_behavior": "Master always mapped 'schedule' to Campaign Helper (campaign scheduling) and ignored chatbot trigger scheduling use case", "correct_behavior": "Without active child context, 'schedule' is ambiguous. With active child: forward to active child.", "correct_master_route": "No active child: ASK_MINIMAL_CLARIFICATION. Active child: forward to active child.", "correct_child_continuity": "Correct child handles scheduling based on clarified context", "correct_state_memory": "Route only after disambiguation", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "How do I schedule it?", "expected_intent": "AMBIGUOUS", "expected_target_agent": null, "expected_master_action": "ASK_MINIMAL_CLARIFICATION", "expected_state_update": {}, "expected_queue_update": null, "expected_handoff": null, "validation_note": "REGRESSION TEST: 'schedule it' with 'it' undefined and no active child context. Must ask: 'Schedule what — a campaign message or a chatbot trigger?' Must NOT default to Campaign Helper." }, { "turn_number": 2, "speaker": "USER", "text": "a campaign blast for tomorrow morning", "expected_intent": "schedule_campaign", "expected_target_agent": "Campaign Helper", "expected_master_action": "ROUTE_TO_CAMPAIGN_HELPER", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Campaign Helper": "ACTIVE", "extracted_params": { "schedule_time": "tomorrow morning" } }, "expected_queue_update": null, "expected_handoff": "Routing to Campaign Helper to schedule your campaign blast.", "validation_note": "After clarification, Campaign Helper correctly handles schedule. Time parameter 'tomorrow morning' extracted." } ] }, { "conversation_id": "H-015", "title": "REGRESSION: Re-asking same question in different child context — no cross-contamination", "tags": ["regression", "context-isolation", "cross-contamination"], "previous_bad_behavior": "Master carried over slot-fill answers from one child session into another child's session, causing wrong parameter values", "correct_behavior": "Each child agent session has isolated parameter context. Switching children clears the previous child's slot state from active extraction context.", "correct_master_route": "New child gets fresh slot context, not inherited from paused child", "correct_child_continuity": "No parameter bleed between children", "correct_state_memory": "Per-child slot state stored separately; switching does not merge contexts", "conversation_steps": [ { "turn_number": 1, "speaker": "USER", "text": "I want to name my template 'summer_sale'", "expected_intent": "set_template_name", "expected_target_agent": "Template Helper", "expected_master_action": "ROUTE_TO_TEMPLATE_HELPER", "expected_state_update": { "active_child_agent": "Template Helper", "child_agent_status_map.Template Helper": "ACTIVE", "template_session.template_name": "summer_sale" }, "expected_queue_update": null, "expected_handoff": "Routing to Template Helper with template name captured.", "validation_note": "Template slot 'template_name=summer_sale' stored in Template Helper session." }, { "turn_number": 2, "speaker": "USER", "text": "Actually, switch to campaign — I want to name my campaign 'summer_sale' too", "expected_intent": "create_campaign_with_name", "expected_target_agent": "Campaign Helper", "expected_master_action": "PRESERVE_CONTEXT_AND_SWITCH", "expected_state_update": { "active_child_agent": "Campaign Helper", "child_agent_status_map.Template Helper": "PAUSED", "campaign_session.campaign_name": "summer_sale" }, "expected_queue_update": null, "expected_handoff": "Switching to Campaign Helper. Template session saved.", "validation_note": "REGRESSION TEST: Campaign session stores campaign_name=summer_sale INDEPENDENTLY of Template session's template_name. No cross-contamination. If Campaign Helper asks 'what is the campaign name?', it should NOT pre-fill from the template name slot — these are separate entity slots in separate sessions." } ] } ] } }, "failure_diagnosis_notes": { "description": "Root cause categories for each type of routing failure. Use these when a test fails to identify the fix location.", "categories": { "FD-001": { "failure_type": "WRONG_CHILD_ROUTE", "description": "Master routes to incorrect child agent", "sub_types": [ { "sub_id": "FD-001-A", "name": "Keyword Match Priority Miss", "symptom": "Low-priority domain keyword fires before high-priority P=115/P=110 rule", "root_cause": "MasterRouteRule priority ordering incorrect, or high-priority rule missing keyword coverage", "fix_location": "modules/masterAgent/scripts/seedWhatsAppBusinessFlowMaster.js — check rule priorities and conditions", "test_cases": ["F-001", "F-002", "H-003", "H-011"] }, { "sub_id": "FD-001-B", "name": "Dependency Check Not Firing", "symptom": "Campaign or Chatbot routed without prerequisite check", "root_cause": "P=110 dependency block rule conditions not matching; or dependency_status not being read from session", "fix_location": "modules/masterAgent/services/routingEngine.service.js — dependency check in rule evaluation", "test_cases": ["H-005", "H-008", "G-010"] }, { "sub_id": "FD-001-C", "name": "Sticky Agent Not Cleared After Completion", "symptom": "Completed child still receives new unrelated requests", "root_cause": "active_child_agent not set to null when child emits COMPLETED status", "fix_location": "modules/masterAgent/services/orchestrationExecutor.service.js — task completion handler", "test_cases": ["H-010"] } ] }, "FD-002": { "failure_type": "CONTEXT_ROUTING_FAILURE", "description": "Master ignores active context when routing", "sub_types": [ { "sub_id": "FD-002-A", "name": "Conflict Rule Not Applied", "symptom": "Template/keyword/publish in active child context causes wrong switch", "root_cause": "Conflict resolution rules (P=75, P=78, P=80) not evaluated before domain keyword rules", "fix_location": "modules/masterAgent/scripts/seedWhatsAppBusinessFlowMaster.js — conflict resolution rules must have higher priority than domain rules", "test_cases": ["F-001", "F-002", "F-009", "H-003", "H-011"] }, { "sub_id": "FD-002-B", "name": "Short Reply Not Forwarded to Active Child", "symptom": "Single-word slot response triggers re-routing or clarification", "root_cause": "Sticky agent bias not applied for short inputs during active slot-fill; P=85 active-child continuation rule not firing", "fix_location": "modules/masterAgent/services/routingEngine.service.js — sticky agent bias logic", "test_cases": ["H-004", "G-005"] }, { "sub_id": "FD-002-C", "name": "Context-Gated Intent Not Winning", "symptom": "Global intent wins over context-gated intent despite active context", "root_cause": "Context boost not applied or applied incorrectly in dialogflow intent matching; shadow intent missing training phrases", "fix_location": "modules/dialogflow/scripts/seedWhatsAppOnboardingHelper.js — ask_eligibility_mid_flow intent; modules/dialogflow/services/intentMatcher.service.js — context boost", "test_cases": ["H-006"] } ] }, "FD-003": { "failure_type": "ENTITY_EXTRACTION_FAILURE", "description": "Parameters not extracted or extracted into wrong slot", "sub_types": [ { "sub_id": "FD-003-A", "name": "Phone Number ID Misclassified", "symptom": "Long numeric string classified as sys.phone instead of phone_number_id", "root_cause": "Phone Number ID entity type not defined as DfEntity or not in snapshot.entities; sys.phone pattern too greedy", "fix_location": "modules/dialogflow/utils/systemEntityData.js — SYSTEM_PATTERNS; seed script for Onboarding Helper", "test_cases": ["H-002"] }, { "sub_id": "FD-003-B", "name": "Multi-Slot Capture Partial", "symptom": "Only first entity extracted when multiple entities present in same utterance", "root_cause": "Entity extractor exits after first match; does not continue scanning for additional entities", "fix_location": "modules/dialogflow/services/entityExtractor.service.js — must iterate all parameter slots, not just first match", "test_cases": ["H-001"] } ] }, "FD-004": { "failure_type": "SESSION_STATE_CORRUPTION", "description": "Session state becomes inconsistent or parameters bleed between children", "sub_types": [ { "sub_id": "FD-004-A", "name": "Cross-Child Parameter Bleed", "symptom": "Slot value from one child prefills slot in different child", "root_cause": "Shared session context not namespaced per child; extractedParams stored at session level not child-session level", "fix_location": "modules/masterAgent/services/orchestrationExecutor.service.js — child session context isolation", "test_cases": ["H-015"] }, { "sub_id": "FD-004-B", "name": "Rapid Switch State Corruption", "symptom": "After 3+ rapid switches, active_child_agent shows wrong child", "root_cause": "State updates not atomic; race condition in child_agent_status_map updates during rapid switches", "fix_location": "modules/masterAgent/services/orchestrationExecutor.service.js — atomic state update pattern", "test_cases": ["G-015"] } ] }, "FD-005": { "failure_type": "SYSTEM_STABILITY", "description": "Master crashes or returns 500 errors on unexpected inputs", "sub_types": [ { "sub_id": "FD-005-A", "name": "Null Input Crash", "symptom": "Empty or emoji-only input causes uncaught exception", "root_cause": "Input validation missing before intent matching pipeline; null/empty check skipped", "fix_location": "modules/masterAgent/controllers/masterChat.controller.js — input sanitization; modules/dialogflow/services/intentMatcher.service.js — null guard", "test_cases": ["G-014", "G-009"] }, { "sub_id": "FD-005-B", "name": "URL/Number Injection Handling", "symptom": "Raw URL or number causes routing to incorrect child", "root_cause": "URL/number pattern matching fires before unrecognized fallback; no validation of input type", "fix_location": "modules/masterAgent/services/routingEngine.service.js — pre-routing input type check", "test_cases": ["G-007", "G-008"] } ] }, "FD-006": { "failure_type": "CLARIFICATION_FAILURE", "description": "Clarification asked when not needed, or not asked when needed", "sub_types": [ { "sub_id": "FD-006-A", "name": "Clarification When Active Child Present", "symptom": "Master asks clarification mid-session instead of forwarding to active child", "root_cause": "Active child check not performed before clarification fallback; P=85 rule not evaluated", "fix_location": "modules/masterAgent/services/routingEngine.service.js — active child priority before clarification fallback", "test_cases": ["F-004", "F-010"] }, { "sub_id": "FD-006-B", "name": "No Clarification for Ambiguous Input", "symptom": "Master blindly routes ambiguous single-word input to wrong child", "root_cause": "Fallback P=10 rule not reaching clarification action; some keyword rule fires instead", "fix_location": "modules/masterAgent/scripts/seedWhatsAppBusinessFlowMaster.js — ensure clarification fallback is lowest priority and covers all non-matched cases", "test_cases": ["G-004", "G-005", "G-006", "F-003"] } ] } } }, "execution_guide": { "description": "How to run, score, and diagnose the QA conversation test suite", "prerequisites": [ "Master agent seeded: node modules/masterAgent/scripts/seedWhatsAppBusinessFlowMaster.js", "All 4 child agents seeded and published", "MongoDB running with test tenant data", "Server running: npm start or node server.js" ], "test_execution_method": { "approach": "Replay each conversation by submitting turns sequentially to the master agent API", "api_endpoint": "POST /ma/chat", "request_format": { "sessionId": "test--", "tenantId": "", "agentId": "", "message": "" }, "session_management": "Use unique sessionId per conversation. Reuse same sessionId across turns within same conversation.", "state_inspection": "After each USER turn, inspect response.data.masterRoute, response.data.targetAgent, response.data.sessionState" }, "scoring_rubric": { "pass_criteria": { "expected_target_agent": "response.data.targetAgent MUST EXACTLY MATCH expected_target_agent (null allowed for clarification turns)", "expected_master_action": "response.data.masterAction MUST EXACTLY MATCH expected_master_action", "expected_state_update_keys": "All keys in expected_state_update must be present in actual session state with matching values", "no_crash": "HTTP 200 must be returned for all inputs. HTTP 500 = automatic fail." }, "partial_pass_criteria": { "expected_handoff": "Handoff message text is evaluated as semantic similarity (not exact match). Score: 1 if core meaning matches.", "expected_queue_update": "Queue contents evaluated by presence of queued task, not exact message text." }, "weights": { "expected_target_agent": 40, "expected_master_action": 30, "expected_state_update": 20, "no_crash": 10 } }, "pass_fail_thresholds": { "conversation_pass": "All turns in conversation score >= 90/100", "category_pass": ">= 90% of conversations in category pass", "suite_pass": ">= 90% of all conversations pass", "regression_category_pass": "100% of H-category conversations MUST pass (regressions are zero-tolerance)" }, "how_to_identify_regressions": { "step1": "Run only Category H tests", "step2": "Any H-category fail = regression. Use conversation's previous_bad_behavior field to verify the OLD bug is occurring", "step3": "Map the failure to FAILURE_DIAGNOSIS_NOTES using the test_cases cross-reference", "step4": "Apply fix in the module identified in fix_location", "step5": "Re-run affected H-category test + all F-category tests with same conflict rule (cross-check)" }, "recommended_run_order": [ "H (Regression) — run first to establish baseline. All must pass.", "F (Conflict/Ambiguous) — run second. Tests routing edge cases.", "G (Adversarial) — run third. Tests system stability.", "A+B+C+D+E (Parts 1+2) — run last for full regression coverage." ], "example_assertions": { "javascript_pseudocode": [ "assert(response.data.targetAgent === step.expected_target_agent, `Turn ${step.turn_number}: Wrong child routed. Expected: ${step.expected_target_agent}, Got: ${response.data.targetAgent}`);", "assert(response.data.masterAction === step.expected_master_action, `Turn ${step.turn_number}: Wrong master action. Expected: ${step.expected_master_action}, Got: ${response.data.masterAction}`);", "assert(response.status === 200, `Turn ${step.turn_number}: Non-200 response. Got: ${response.status}`);", "if (step.expected_state_update.active_child_agent !== undefined) { assert(session.active_child_agent === step.expected_state_update.active_child_agent, `Turn ${step.turn_number}: Wrong active_child_agent`); }" ] }, "debug_tips": [ "If H-006 fails: check if ask_eligibility_mid_flow intent exists in seedWhatsAppOnboardingHelper.js and has inputContexts: ['onboarding-active']", "If H-003/H-011 fail: verify P=80 conflict rule exists in seedWhatsAppBusinessFlowMaster.js with condition checking active_child_agent === 'Campaign Helper'", "If G-004/G-005 fail: check P=10 clarification fallback is last rule and catchesambiguous single-word inputs", "If H-010 fails: check orchestrationExecutor sets active_child_agent=null when child emits COMPLETED status", "If G-015 fails (state corruption): check atomic state update pattern in orchestrationExecutor — all child_agent_status_map updates must complete before routing proceeds", "If any G-00x crashes with 500: check input validation in masterChat.controller.js — add null/empty/non-text guard before pipeline entry" ] } }