{ "capturedAt": "2026-07-14T17:57:22.063Z", "environment": { "node": "v22.23.1", "platform": "darwin", "arch": "arm64" }, "protocol": { "name": "agentic-rag-answer-decision-eval", "version": "1.0.0", "methods": [ "fixed-one-shot", "adaptive-stateful" ], "budget": { "maxRetrievalSteps": 2, "maxTransitions": 9, "topK": 3 }, "supportThreshold": 4, "aclRule": "filter by authenticated tenant before scoring; any retrieved unauthorized document is a hard failure", "scopeRule": "collect every exact case-insensitive known-tenant mention; stop before retrieval when any mention differs from the authenticated tenant", "acceptanceRule": "exact answer/abstain decision and answer facts, no ACL leakage, retrieval steps within budget", "latencyWorkUnitModel": { "route": 1, "retrieveBase": 3, "assess": 1, "rewrite": 1, "decide": 1, "scopeGuard": 1 }, "costMilliunitModel": { "route": 100, "retrieveBase": 500, "retrieveCandidate": 50, "assess": 100, "rewrite": 150, "decide": 150, "scopeGuard": 100 } }, "dataset": { "corpusDocuments": 39, "tenants": [ "atlas", "boreal", "cinder" ], "queries": 24, "answerableQueries": 18, "abstentionQueries": 6 }, "methods": { "fixedOneShot": { "summary": { "queries": 24, "route": { "correct": 22, "denominator": 23, "accuracy": 0.956522 }, "relevantRecall": { "recovered": 14, "denominator": 19, "microRecall": 0.736842, "answerableQueriesWithAllRelevant": 13, "answerableQueryDenominator": 18, "answerableQueryRecall": 0.722222 }, "acl": { "leakageQueries": 0, "denominator": 24, "leakageRate": 0, "hardFailure": false }, "abstention": { "expected": 6, "actual": 10, "correct": 4, "precision": 0.4, "recall": 0.666667, "accuracy": 0.666667 }, "answerDecision": { "accepted": 15, "denominator": 24, "accuracy": 0.625 }, "budget": { "averageRetrievalSteps": 0.958333, "maxRetrievalStepsObserved": 1, "retrievalBudgetStops": 0, "transitionBudgetStops": 0, "loopStops": 0, "withinBudget": 24, "denominator": 24 }, "efficiency": { "totalLatencyWorkUnits": 187, "averageLatencyWorkUnits": 7.791667, "totalCostUnits": 21.05, "unitAcceptedCost": 1.403333 }, "failures": [ { "queryId": "q01-atlas-supper", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q04-atlas-pto", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-people-leave" ] }, { "queryId": "q10-atlas-throttle", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-reliability-rate-limit" ] }, { "queryId": "q12-cinder-severity", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "cinder-reliability-incident" ] }, { "queryId": "q13-atlas-erasure", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-governance-customer-retention" ] }, { "queryId": "q14-boreal-former-worker", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q21-atlas-composite", "reasons": [ "answer-content-mismatch" ], "missedRelevant": [ "atlas-access-offboarding" ] }, { "queryId": "q23-cinder-p1-log-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q24-atlas-residency-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] } ], "scores": [ { "queryId": "q01-atlas-supper", "routeCorrect": true, "retrievedIds": [ "atlas-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q02-boreal-dinner", "routeCorrect": true, "retrievedIds": [ "boreal-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q03-cinder-receipt", "routeCorrect": true, "retrievedIds": [ "cinder-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q04-atlas-pto", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 1, "relevantRecovered": 0, "missedRelevant": [ "atlas-people-leave" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q05-boreal-vacation", "routeCorrect": true, "retrievedIds": [ "boreal-people-leave" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q06-cinder-parental", "routeCorrect": true, "retrievedIds": [ "cinder-people-leave", "cinder-people-parental" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q07-atlas-mfa", "routeCorrect": true, "retrievedIds": [ "atlas-security-mfa" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q08-boreal-totp", "routeCorrect": true, "retrievedIds": [ "boreal-security-mfa" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q09-cinder-secret", "routeCorrect": true, "retrievedIds": [ "cinder-security-secret" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q10-atlas-throttle", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 1, "relevantRecovered": 0, "missedRelevant": [ "atlas-reliability-rate-limit" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q11-boreal-429", "routeCorrect": true, "retrievedIds": [ "boreal-reliability-rate-limit" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q12-cinder-severity", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 1, "relevantRecovered": 0, "missedRelevant": [ "cinder-reliability-incident" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q13-atlas-erasure", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 1, "relevantRecovered": 0, "missedRelevant": [ "atlas-governance-customer-retention" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q14-boreal-former-worker", "routeCorrect": true, "retrievedIds": [ "boreal-governance-audit-retention" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "abstain", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q15-cinder-deletion", "routeCorrect": true, "retrievedIds": [ "cinder-governance-customer-retention" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q16-atlas-onboarding", "routeCorrect": false, "retrievedIds": [ "atlas-access-offboarding", "atlas-access-onboarding" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q17-boreal-production", "routeCorrect": true, "retrievedIds": [ "boreal-access-production" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q18-atlas-dental", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q19-cross-tenant", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q20-cinder-parking", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q21-atlas-composite", "routeCorrect": true, "retrievedIds": [ "atlas-governance-audit-retention" ], "relevantExpected": 2, "relevantRecovered": 1, "missedRelevant": [ "atlas-access-offboarding" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": false, "accepted": false, "failureReasons": [ "answer-content-mismatch" ] }, { "queryId": "q22-boreal-ambiguous", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q23-cinder-p1-log-gap", "routeCorrect": true, "retrievedIds": [ "cinder-reliability-incident" ], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "answer", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q24-atlas-residency-gap", "routeCorrect": true, "retrievedIds": [ "atlas-governance-customer-retention" ], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "answer", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] } ] }, "runs": [ { "queryId": "q01-atlas-supper", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "atlas-finance-meal", "score": 1 } ], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "What is the client supper ceiling?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "What is the client supper ceiling?", "candidateCount": 2, "ranking": [ { "id": "atlas-finance-meal", "score": 1 } ] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q02-boreal-dinner", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-finance-meal", "score": 11 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-meal-cap-120" ], "sourceIds": [ "boreal-finance-meal" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "How much can I expense for a customer dinner?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "How much can I expense for a customer dinner?", "candidateCount": 2, "ranking": [ { "id": "boreal-finance-meal", "score": 11 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-meal-cap-120" ], "sourceIds": [ "boreal-finance-meal" ] } } ] }, { "queryId": "q03-cinder-receipt", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "cinder-finance-meal", "score": 18 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-meal-receipt-cap-70" ], "sourceIds": [ "cinder-finance-meal" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Is a receipt required for a client meal?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "Is a receipt required for a client meal?", "candidateCount": 2, "ranking": [ { "id": "cinder-finance-meal", "score": 18 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-meal-receipt-cap-70" ], "sourceIds": [ "cinder-finance-meal" ] } } ] }, { "queryId": "q04-atlas-pto", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Can PTO roll into next year?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "Can PTO roll into next year?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q05-boreal-vacation", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-people-leave", "score": 11 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-leave-carryover-3" ], "sourceIds": [ "boreal-people-leave" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "How many vacation days carry over?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "How many vacation days carry over?", "candidateCount": 2, "ranking": [ { "id": "boreal-people-leave", "score": 11 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-leave-carryover-3" ], "sourceIds": [ "boreal-people-leave" ] } } ] }, { "queryId": "q06-cinder-parental", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "cinder-people-parental", "score": 20 }, { "id": "cinder-people-leave", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-parental-20w" ], "sourceIds": [ "cinder-people-parental" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "What is the paid parental leave duration?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "What is the paid parental leave duration?", "candidateCount": 2, "ranking": [ { "id": "cinder-people-parental", "score": 20 }, { "id": "cinder-people-leave", "score": 6 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-parental-20w" ], "sourceIds": [ "cinder-people-parental" ] } } ] }, { "queryId": "q07-atlas-mfa", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "atlas-security-mfa", "score": 13 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-mfa-two-support-approvals" ], "sourceIds": [ "atlas-security-mfa" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "At Atlas I lost my phone and authenticator. How is MFA recovery approved?", "route": "security", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "security", "text": "At Atlas I lost my phone and authenticator. How is MFA recovery approved?", "candidateCount": 2, "ranking": [ { "id": "atlas-security-mfa", "score": 13 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-mfa-two-support-approvals" ], "sourceIds": [ "atlas-security-mfa" ] } } ] }, { "queryId": "q08-boreal-totp", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-security-mfa", "score": 18 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-mfa-code-manager-attestation" ], "sourceIds": [ "boreal-security-mfa" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Can Boreal support disclose the TOTP seed during recovery?", "route": "security", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "security", "text": "Can Boreal support disclose the TOTP seed during recovery?", "candidateCount": 2, "ranking": [ { "id": "boreal-security-mfa", "score": 18 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-mfa-code-manager-attestation" ], "sourceIds": [ "boreal-security-mfa" ] } } ] }, { "queryId": "q09-cinder-secret", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "cinder-security-secret", "score": 21 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-secret-rotation-30d" ], "sourceIds": [ "cinder-security-secret" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "What is the cadence for production secret rotation?", "route": "security", "confidence": 0.666667 }, { "type": "retrieve", "step": 1, "route": "security", "text": "What is the cadence for production secret rotation?", "candidateCount": 2, "ranking": [ { "id": "cinder-security-secret", "score": 21 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-secret-rotation-30d" ], "sourceIds": [ "cinder-security-secret" ] } } ] }, { "queryId": "q10-atlas-throttle", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "What should a client do when the endpoint is throttled?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "What should a client do when the endpoint is throttled?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q11-boreal-429", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-reliability-rate-limit", "score": 12 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-429-retry-after-two" ], "sourceIds": [ "boreal-reliability-rate-limit" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "For HTTP 429, how many retries and which header?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "For HTTP 429, how many retries and which header?", "candidateCount": 2, "ranking": [ { "id": "boreal-reliability-rate-limit", "score": 12 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-429-retry-after-two" ], "sourceIds": [ "boreal-reliability-rate-limit" ] } } ] }, { "queryId": "q12-cinder-severity", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Who is called for a severity-one outage?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "Who is called for a severity-one outage?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q13-atlas-erasure", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "How long is the erasure window after an agreement ends?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "How long is the erasure window after an agreement ends?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q14-boreal-former-worker", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-governance-audit-retention", "score": 3 } ], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "How long are former worker audit records kept?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "How long are former worker audit records kept?", "candidateCount": 2, "ranking": [ { "id": "boreal-governance-audit-retention", "score": 3 } ] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q15-cinder-deletion", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "cinder-governance-customer-retention", "score": 15 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-customer-delete-14d" ], "sourceIds": [ "cinder-governance-customer-retention" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "When is customer data deleted after contract termination?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "When is customer data deleted after contract termination?", "candidateCount": 2, "ranking": [ { "id": "cinder-governance-customer-retention", "score": 15 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-customer-delete-14d" ], "sourceIds": [ "cinder-governance-customer-retention" ] } } ] }, { "queryId": "q16-atlas-onboarding", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": null, "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "atlas-access-onboarding", "score": 17 }, { "id": "atlas-access-offboarding", "score": 5 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-onboarding-idp-ticket-readonly" ], "sourceIds": [ "atlas-access-onboarding" ] }, "workUnits": 18, "costUnits": 1.4, "trace": [ { "type": "route", "text": "Which access systems does a new hire get on day one?", "route": null, "confidence": 0 }, { "type": "retrieve", "step": 1, "route": null, "text": "Which access systems does a new hire get on day one?", "candidateCount": 13, "ranking": [ { "id": "atlas-access-onboarding", "score": 17 }, { "id": "atlas-access-offboarding", "score": 5 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-onboarding-idp-ticket-readonly" ], "sourceIds": [ "atlas-access-onboarding" ] } } ] }, { "queryId": "q17-boreal-production", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": "access", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "boreal-access-production", "score": 16 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-production-write-ticket-owner" ], "sourceIds": [ "boreal-access-production" ] }, "workUnits": 8, "costUnits": 0.9, "trace": [ { "type": "route", "text": "Can an engineer self-approve a production write?", "route": "access", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "access", "text": "Can an engineer self-approve a production write?", "candidateCount": 3, "ranking": [ { "id": "boreal-access-production", "score": 16 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-production-write-ticket-owner" ], "sourceIds": [ "boreal-access-production" ] } } ] }, { "queryId": "q18-atlas-dental", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Does Atlas cover pet dental insurance?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "Does Atlas cover pet dental insurance?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q19-cross-tenant", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "finance", "retrievalSteps": 0, "transitionCount": 2, "stoppedBy": "scope-violation", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "tenant-scope-mismatch", "answerKeys": [] }, "workUnits": 3, "costUnits": 0.35, "trace": [ { "type": "route", "text": "What is Boreal's customer dinner allowance?", "route": "finance", "confidence": 1 }, { "type": "scope-guard", "mentionedTenants": [ "boreal" ], "foreignTenants": [ "boreal" ], "authenticatedTenant": "atlas", "allowed": false }, { "type": "decision", "outcome": { "type": "abstain", "reason": "tenant-scope-mismatch", "answerKeys": [] } } ] }, { "queryId": "q20-cinder-parking", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "What is the office parking stipend?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "What is the office parking stipend?", "candidateCount": 2, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q21-atlas-composite", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "atlas-governance-audit-retention", "score": 9 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-audit-retention-365d" ], "sourceIds": [ "atlas-governance-audit-retention" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "After an employee leaves, disable access and retain audit logs how long?", "route": "governance", "confidence": 0.666667 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "After an employee leaves, disable access and retain audit logs how long?", "candidateCount": 2, "ranking": [ { "id": "atlas-governance-audit-retention", "score": 9 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-audit-retention-365d" ], "sourceIds": [ "atlas-governance-audit-retention" ] } } ] }, { "queryId": "q22-boreal-ambiguous", "tenantId": "boreal", "method": "fixed-one-shot", "finalRoute": null, "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 18, "costUnits": 1.4, "trace": [ { "type": "route", "text": "What is the limit?", "route": null, "confidence": 0 }, { "type": "retrieve", "step": 1, "route": null, "text": "What is the limit?", "candidateCount": 13, "ranking": [] }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q23-cinder-p1-log-gap", "tenantId": "cinder", "method": "fixed-one-shot", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "cinder-reliability-incident", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "How long do logs persist after a P1?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "How long do logs persist after a P1?", "candidateCount": 2, "ranking": [ { "id": "cinder-reliability-incident", "score": 6 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] } } ] }, { "queryId": "q24-atlas-residency-gap", "tenantId": "atlas", "method": "fixed-one-shot", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 2, "stoppedBy": "one-shot-complete", "withinBudget": true, "retrieved": [ { "id": "atlas-governance-customer-retention", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] }, "workUnits": 7, "costUnits": 0.85, "trace": [ { "type": "route", "text": "Where is EU payroll data residency?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "Where is EU payroll data residency?", "candidateCount": 2, "ranking": [ { "id": "atlas-governance-customer-retention", "score": 6 } ] }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] } } ] } ] }, "adaptiveStateful": { "summary": { "queries": 24, "route": { "correct": 22, "denominator": 23, "accuracy": 0.956522 }, "relevantRecall": { "recovered": 18, "denominator": 19, "microRecall": 0.947368, "answerableQueriesWithAllRelevant": 17, "answerableQueryDenominator": 18, "answerableQueryRecall": 0.944444 }, "acl": { "leakageQueries": 0, "denominator": 24, "leakageRate": 0, "hardFailure": false }, "abstention": { "expected": 6, "actual": 4, "correct": 4, "precision": 1, "recall": 0.666667, "accuracy": 0.916667 }, "answerDecision": { "accepted": 21, "denominator": 24, "accuracy": 0.875 }, "budget": { "averageRetrievalSteps": 1.208333, "maxRetrievalStepsObserved": 2, "retrievalBudgetStops": 0, "transitionBudgetStops": 0, "loopStops": 0, "withinBudget": 24, "denominator": 24 }, "efficiency": { "totalLatencyWorkUnits": 262, "averageLatencyWorkUnits": 10.916667, "totalCostUnits": 29.65, "unitAcceptedCost": 1.411905 }, "failures": [ { "queryId": "q21-atlas-composite", "reasons": [ "answer-content-mismatch" ], "missedRelevant": [ "atlas-access-offboarding" ] }, { "queryId": "q23-cinder-p1-log-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q24-atlas-residency-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] } ], "scores": [ { "queryId": "q01-atlas-supper", "routeCorrect": true, "retrievedIds": [ "atlas-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q02-boreal-dinner", "routeCorrect": true, "retrievedIds": [ "boreal-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q03-cinder-receipt", "routeCorrect": true, "retrievedIds": [ "cinder-finance-meal" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q04-atlas-pto", "routeCorrect": true, "retrievedIds": [ "atlas-people-leave", "atlas-people-parental" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q05-boreal-vacation", "routeCorrect": true, "retrievedIds": [ "boreal-people-leave" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q06-cinder-parental", "routeCorrect": true, "retrievedIds": [ "cinder-people-leave", "cinder-people-parental" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q07-atlas-mfa", "routeCorrect": true, "retrievedIds": [ "atlas-security-mfa" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q08-boreal-totp", "routeCorrect": true, "retrievedIds": [ "boreal-security-mfa" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q09-cinder-secret", "routeCorrect": true, "retrievedIds": [ "cinder-security-secret" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q10-atlas-throttle", "routeCorrect": true, "retrievedIds": [ "atlas-reliability-rate-limit" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q11-boreal-429", "routeCorrect": true, "retrievedIds": [ "boreal-reliability-rate-limit" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q12-cinder-severity", "routeCorrect": true, "retrievedIds": [ "cinder-reliability-incident" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q13-atlas-erasure", "routeCorrect": true, "retrievedIds": [ "atlas-governance-audit-retention", "atlas-governance-customer-retention" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q14-boreal-former-worker", "routeCorrect": true, "retrievedIds": [ "boreal-governance-audit-retention", "boreal-governance-customer-retention" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q15-cinder-deletion", "routeCorrect": true, "retrievedIds": [ "cinder-governance-customer-retention" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q16-atlas-onboarding", "routeCorrect": false, "retrievedIds": [ "atlas-access-offboarding", "atlas-access-onboarding" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q17-boreal-production", "routeCorrect": true, "retrievedIds": [ "boreal-access-production" ], "relevantExpected": 1, "relevantRecovered": 1, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q18-atlas-dental", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q19-cross-tenant", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q20-cinder-parking", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q21-atlas-composite", "routeCorrect": true, "retrievedIds": [ "atlas-governance-audit-retention" ], "relevantExpected": 2, "relevantRecovered": 1, "missedRelevant": [ "atlas-access-offboarding" ], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "answer", "actualDecision": "answer", "decisionTypeCorrect": true, "answerCorrect": false, "accepted": false, "failureReasons": [ "answer-content-mismatch" ] }, { "queryId": "q22-boreal-ambiguous", "routeCorrect": true, "retrievedIds": [], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "abstain", "decisionTypeCorrect": true, "answerCorrect": true, "accepted": true, "failureReasons": [] }, { "queryId": "q23-cinder-p1-log-gap", "routeCorrect": true, "retrievedIds": [ "cinder-reliability-incident" ], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "answer", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] }, { "queryId": "q24-atlas-residency-gap", "routeCorrect": true, "retrievedIds": [ "atlas-governance-customer-retention" ], "relevantExpected": 0, "relevantRecovered": 0, "missedRelevant": [], "aclLeakage": { "hardFailure": false, "leakedIds": [] }, "expectedDecision": "abstain", "actualDecision": "answer", "decisionTypeCorrect": false, "answerCorrect": false, "accepted": false, "failureReasons": [ "decision-type-mismatch" ] } ] }, "runs": [ { "queryId": "q01-atlas-supper", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "finance", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-finance-meal", "score": 10 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-meal-cap-80" ], "sourceIds": [ "atlas-finance-meal" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "What is the client supper ceiling?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "What is the client supper ceiling?", "candidateCount": 2, "ranking": [ { "id": "atlas-finance-meal", "score": 1 } ] }, { "type": "assess", "supported": false, "topScore": 1, "retrievalSteps": 1 }, { "type": "rewrite", "from": "What is the client supper ceiling?", "to": "What is the client supper ceiling? meal dinner limit cap", "changed": true }, { "type": "route", "text": "What is the client supper ceiling? meal dinner limit cap", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "finance", "text": "What is the client supper ceiling? meal dinner limit cap", "candidateCount": 2, "ranking": [ { "id": "atlas-finance-meal", "score": 10 } ] }, { "type": "assess", "supported": true, "topScore": 10, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-meal-cap-80" ], "sourceIds": [ "atlas-finance-meal" ] } } ] }, { "queryId": "q02-boreal-dinner", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-finance-meal", "score": 11 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-meal-cap-120" ], "sourceIds": [ "boreal-finance-meal" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "How much can I expense for a customer dinner?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "How much can I expense for a customer dinner?", "candidateCount": 2, "ranking": [ { "id": "boreal-finance-meal", "score": 11 } ] }, { "type": "assess", "supported": true, "topScore": 11, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-meal-cap-120" ], "sourceIds": [ "boreal-finance-meal" ] } } ] }, { "queryId": "q03-cinder-receipt", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-finance-meal", "score": 18 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-meal-receipt-cap-70" ], "sourceIds": [ "cinder-finance-meal" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "Is a receipt required for a client meal?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "Is a receipt required for a client meal?", "candidateCount": 2, "ranking": [ { "id": "cinder-finance-meal", "score": 18 } ] }, { "type": "assess", "supported": true, "topScore": 18, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-meal-receipt-cap-70" ], "sourceIds": [ "cinder-finance-meal" ] } } ] }, { "queryId": "q04-atlas-pto", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "people", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-people-leave", "score": 15 }, { "id": "atlas-people-parental", "score": 4 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-leave-carryover-5" ], "sourceIds": [ "atlas-people-leave" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "Can PTO roll into next year?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "Can PTO roll into next year?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "Can PTO roll into next year?", "to": "Can PTO roll into next year? annual leave carryover carryover", "changed": true }, { "type": "route", "text": "Can PTO roll into next year? annual leave carryover carryover", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "people", "text": "Can PTO roll into next year? annual leave carryover carryover", "candidateCount": 2, "ranking": [ { "id": "atlas-people-leave", "score": 15 }, { "id": "atlas-people-parental", "score": 4 } ] }, { "type": "assess", "supported": true, "topScore": 15, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-leave-carryover-5" ], "sourceIds": [ "atlas-people-leave" ] } } ] }, { "queryId": "q05-boreal-vacation", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-people-leave", "score": 11 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-leave-carryover-3" ], "sourceIds": [ "boreal-people-leave" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "How many vacation days carry over?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "How many vacation days carry over?", "candidateCount": 2, "ranking": [ { "id": "boreal-people-leave", "score": 11 } ] }, { "type": "assess", "supported": true, "topScore": 11, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-leave-carryover-3" ], "sourceIds": [ "boreal-people-leave" ] } } ] }, { "queryId": "q06-cinder-parental", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-people-parental", "score": 20 }, { "id": "cinder-people-leave", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-parental-20w" ], "sourceIds": [ "cinder-people-parental" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "What is the paid parental leave duration?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "What is the paid parental leave duration?", "candidateCount": 2, "ranking": [ { "id": "cinder-people-parental", "score": 20 }, { "id": "cinder-people-leave", "score": 6 } ] }, { "type": "assess", "supported": true, "topScore": 20, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-parental-20w" ], "sourceIds": [ "cinder-people-parental" ] } } ] }, { "queryId": "q07-atlas-mfa", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-security-mfa", "score": 13 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-mfa-two-support-approvals" ], "sourceIds": [ "atlas-security-mfa" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "At Atlas I lost my phone and authenticator. How is MFA recovery approved?", "route": "security", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "security", "text": "At Atlas I lost my phone and authenticator. How is MFA recovery approved?", "candidateCount": 2, "ranking": [ { "id": "atlas-security-mfa", "score": 13 } ] }, { "type": "assess", "supported": true, "topScore": 13, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-mfa-two-support-approvals" ], "sourceIds": [ "atlas-security-mfa" ] } } ] }, { "queryId": "q08-boreal-totp", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-security-mfa", "score": 18 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-mfa-code-manager-attestation" ], "sourceIds": [ "boreal-security-mfa" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "Can Boreal support disclose the TOTP seed during recovery?", "route": "security", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "security", "text": "Can Boreal support disclose the TOTP seed during recovery?", "candidateCount": 2, "ranking": [ { "id": "boreal-security-mfa", "score": 18 } ] }, { "type": "assess", "supported": true, "topScore": 18, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-mfa-code-manager-attestation" ], "sourceIds": [ "boreal-security-mfa" ] } } ] }, { "queryId": "q09-cinder-secret", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "security", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-security-secret", "score": 21 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-secret-rotation-30d" ], "sourceIds": [ "cinder-security-secret" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "What is the cadence for production secret rotation?", "route": "security", "confidence": 0.666667 }, { "type": "retrieve", "step": 1, "route": "security", "text": "What is the cadence for production secret rotation?", "candidateCount": 2, "ranking": [ { "id": "cinder-security-secret", "score": 21 } ] }, { "type": "assess", "supported": true, "topScore": 21, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-secret-rotation-30d" ], "sourceIds": [ "cinder-security-secret" ] } } ] }, { "queryId": "q10-atlas-throttle", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "reliability", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-reliability-rate-limit", "score": 16 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-429-jitter-four-attempts" ], "sourceIds": [ "atlas-reliability-rate-limit" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "What should a client do when the endpoint is throttled?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "What should a client do when the endpoint is throttled?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "What should a client do when the endpoint is throttled?", "to": "What should a client do when the endpoint is throttled? API 429 Retry-After backoff", "changed": true }, { "type": "route", "text": "What should a client do when the endpoint is throttled? API 429 Retry-After backoff", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "reliability", "text": "What should a client do when the endpoint is throttled? API 429 Retry-After backoff", "candidateCount": 2, "ranking": [ { "id": "atlas-reliability-rate-limit", "score": 16 } ] }, { "type": "assess", "supported": true, "topScore": 16, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-429-jitter-four-attempts" ], "sourceIds": [ "atlas-reliability-rate-limit" ] } } ] }, { "queryId": "q11-boreal-429", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-reliability-rate-limit", "score": 12 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-429-retry-after-two" ], "sourceIds": [ "boreal-reliability-rate-limit" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "For HTTP 429, how many retries and which header?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "For HTTP 429, how many retries and which header?", "candidateCount": 2, "ranking": [ { "id": "boreal-reliability-rate-limit", "score": 12 } ] }, { "type": "assess", "supported": true, "topScore": 12, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-429-retry-after-two" ], "sourceIds": [ "boreal-reliability-rate-limit" ] } } ] }, { "queryId": "q12-cinder-severity", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "reliability", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-reliability-incident", "score": 12 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "Who is called for a severity-one outage?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "Who is called for a severity-one outage?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "Who is called for a severity-one outage?", "to": "Who is called for a severity-one outage? P1 incident commander", "changed": true }, { "type": "route", "text": "Who is called for a severity-one outage? P1 incident commander", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "reliability", "text": "Who is called for a severity-one outage? P1 incident commander", "candidateCount": 2, "ranking": [ { "id": "cinder-reliability-incident", "score": 12 } ] }, { "type": "assess", "supported": true, "topScore": 12, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] } } ] }, { "queryId": "q13-atlas-erasure", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "governance", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-governance-customer-retention", "score": 28 }, { "id": "atlas-governance-audit-retention", "score": 5 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "How long is the erasure window after an agreement ends?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "How long is the erasure window after an agreement ends?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "How long is the erasure window after an agreement ends?", "to": "How long is the erasure window after an agreement ends? customer data deletion retention contract termination", "changed": true }, { "type": "route", "text": "How long is the erasure window after an agreement ends? customer data deletion retention contract termination", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "governance", "text": "How long is the erasure window after an agreement ends? customer data deletion retention contract termination", "candidateCount": 2, "ranking": [ { "id": "atlas-governance-customer-retention", "score": 28 }, { "id": "atlas-governance-audit-retention", "score": 5 } ] }, { "type": "assess", "supported": true, "topScore": 28, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] } } ] }, { "queryId": "q14-boreal-former-worker", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "governance", "retrievalSteps": 2, "transitionCount": 7, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-governance-audit-retention", "score": 15 }, { "id": "boreal-governance-customer-retention", "score": 5 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-audit-retention-7y" ], "sourceIds": [ "boreal-governance-audit-retention" ] }, "workUnits": 16, "costUnits": 1.9, "trace": [ { "type": "route", "text": "How long are former worker audit records kept?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "How long are former worker audit records kept?", "candidateCount": 2, "ranking": [ { "id": "boreal-governance-audit-retention", "score": 3 } ] }, { "type": "assess", "supported": false, "topScore": 3, "retrievalSteps": 1 }, { "type": "rewrite", "from": "How long are former worker audit records kept?", "to": "How long are former worker audit records kept? employee offboarding audit log retention", "changed": true }, { "type": "route", "text": "How long are former worker audit records kept? employee offboarding audit log retention", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 2, "route": "governance", "text": "How long are former worker audit records kept? employee offboarding audit log retention", "candidateCount": 2, "ranking": [ { "id": "boreal-governance-audit-retention", "score": 15 }, { "id": "boreal-governance-customer-retention", "score": 5 } ] }, { "type": "assess", "supported": true, "topScore": 15, "retrievalSteps": 2 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-audit-retention-7y" ], "sourceIds": [ "boreal-governance-audit-retention" ] } } ] }, { "queryId": "q15-cinder-deletion", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-governance-customer-retention", "score": 15 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-customer-delete-14d" ], "sourceIds": [ "cinder-governance-customer-retention" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "When is customer data deleted after contract termination?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "When is customer data deleted after contract termination?", "candidateCount": 2, "ranking": [ { "id": "cinder-governance-customer-retention", "score": 15 } ] }, { "type": "assess", "supported": true, "topScore": 15, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-customer-delete-14d" ], "sourceIds": [ "cinder-governance-customer-retention" ] } } ] }, { "queryId": "q16-atlas-onboarding", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": null, "retrievalSteps": 1, "transitionCount": 4, "stoppedBy": "no-rewrite", "withinBudget": true, "retrieved": [ { "id": "atlas-access-onboarding", "score": 17 }, { "id": "atlas-access-offboarding", "score": 5 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-onboarding-idp-ticket-readonly" ], "sourceIds": [ "atlas-access-onboarding" ] }, "workUnits": 20, "costUnits": 1.65, "trace": [ { "type": "route", "text": "Which access systems does a new hire get on day one?", "route": null, "confidence": 0 }, { "type": "retrieve", "step": 1, "route": null, "text": "Which access systems does a new hire get on day one?", "candidateCount": 13, "ranking": [ { "id": "atlas-access-onboarding", "score": 17 }, { "id": "atlas-access-offboarding", "score": 5 } ] }, { "type": "assess", "supported": false, "topScore": 17, "retrievalSteps": 1 }, { "type": "rewrite", "from": "Which access systems does a new hire get on day one?", "to": "Which access systems does a new hire get on day one?", "changed": false }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-onboarding-idp-ticket-readonly" ], "sourceIds": [ "atlas-access-onboarding" ] } } ] }, { "queryId": "q17-boreal-production", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": "access", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "boreal-access-production", "score": 16 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-production-write-ticket-owner" ], "sourceIds": [ "boreal-access-production" ] }, "workUnits": 9, "costUnits": 1, "trace": [ { "type": "route", "text": "Can an engineer self-approve a production write?", "route": "access", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "access", "text": "Can an engineer self-approve a production write?", "candidateCount": 3, "ranking": [ { "id": "boreal-access-production", "score": 16 } ] }, { "type": "assess", "supported": true, "topScore": 16, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "boreal-production-write-ticket-owner" ], "sourceIds": [ "boreal-access-production" ] } } ] }, { "queryId": "q18-atlas-dental", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "people", "retrievalSteps": 1, "transitionCount": 4, "stoppedBy": "no-rewrite", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 9, "costUnits": 1.1, "trace": [ { "type": "route", "text": "Does Atlas cover pet dental insurance?", "route": "people", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "people", "text": "Does Atlas cover pet dental insurance?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "Does Atlas cover pet dental insurance?", "to": "Does Atlas cover pet dental insurance?", "changed": false }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q19-cross-tenant", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "finance", "retrievalSteps": 0, "transitionCount": 2, "stoppedBy": "scope-violation", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "tenant-scope-mismatch", "answerKeys": [] }, "workUnits": 3, "costUnits": 0.35, "trace": [ { "type": "route", "text": "What is Boreal's customer dinner allowance?", "route": "finance", "confidence": 1 }, { "type": "scope-guard", "mentionedTenants": [ "boreal" ], "foreignTenants": [ "boreal" ], "authenticatedTenant": "atlas", "allowed": false }, { "type": "decision", "outcome": { "type": "abstain", "reason": "tenant-scope-mismatch", "answerKeys": [] } } ] }, { "queryId": "q20-cinder-parking", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "finance", "retrievalSteps": 1, "transitionCount": 4, "stoppedBy": "no-rewrite", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 9, "costUnits": 1.1, "trace": [ { "type": "route", "text": "What is the office parking stipend?", "route": "finance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "finance", "text": "What is the office parking stipend?", "candidateCount": 2, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "What is the office parking stipend?", "to": "What is the office parking stipend?", "changed": false }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q21-atlas-composite", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-governance-audit-retention", "score": 9 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-audit-retention-365d" ], "sourceIds": [ "atlas-governance-audit-retention" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "After an employee leaves, disable access and retain audit logs how long?", "route": "governance", "confidence": 0.666667 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "After an employee leaves, disable access and retain audit logs how long?", "candidateCount": 2, "ranking": [ { "id": "atlas-governance-audit-retention", "score": 9 } ] }, { "type": "assess", "supported": true, "topScore": 9, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-audit-retention-365d" ], "sourceIds": [ "atlas-governance-audit-retention" ] } } ] }, { "queryId": "q22-boreal-ambiguous", "tenantId": "boreal", "method": "adaptive-stateful", "finalRoute": null, "retrievalSteps": 1, "transitionCount": 4, "stoppedBy": "no-rewrite", "withinBudget": true, "retrieved": [], "decision": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] }, "workUnits": 20, "costUnits": 1.65, "trace": [ { "type": "route", "text": "What is the limit?", "route": null, "confidence": 0 }, { "type": "retrieve", "step": 1, "route": null, "text": "What is the limit?", "candidateCount": 13, "ranking": [] }, { "type": "assess", "supported": false, "topScore": 0, "retrievalSteps": 1 }, { "type": "rewrite", "from": "What is the limit?", "to": "What is the limit?", "changed": false }, { "type": "decision", "outcome": { "type": "abstain", "reason": "insufficient-support", "answerKeys": [] } } ] }, { "queryId": "q23-cinder-p1-log-gap", "tenantId": "cinder", "method": "adaptive-stateful", "finalRoute": "reliability", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "cinder-reliability-incident", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "How long do logs persist after a P1?", "route": "reliability", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "reliability", "text": "How long do logs persist after a P1?", "candidateCount": 2, "ranking": [ { "id": "cinder-reliability-incident", "score": 6 } ] }, { "type": "assess", "supported": true, "topScore": 6, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "cinder-p1-commander-immediate" ], "sourceIds": [ "cinder-reliability-incident" ] } } ] }, { "queryId": "q24-atlas-residency-gap", "tenantId": "atlas", "method": "adaptive-stateful", "finalRoute": "governance", "retrievalSteps": 1, "transitionCount": 3, "stoppedBy": "support-sufficient", "withinBudget": true, "retrieved": [ { "id": "atlas-governance-customer-retention", "score": 6 } ], "decision": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] }, "workUnits": 8, "costUnits": 0.95, "trace": [ { "type": "route", "text": "Where is EU payroll data residency?", "route": "governance", "confidence": 1 }, { "type": "retrieve", "step": 1, "route": "governance", "text": "Where is EU payroll data residency?", "candidateCount": 2, "ranking": [ { "id": "atlas-governance-customer-retention", "score": 6 } ] }, { "type": "assess", "supported": true, "topScore": 6, "retrievalSteps": 1 }, { "type": "decision", "outcome": { "type": "answer", "reason": "supported", "answerKeys": [ "atlas-customer-delete-30d" ], "sourceIds": [ "atlas-governance-customer-retention" ] } } ] } ] } }, "comparison": { "answerDecisionAccuracyDelta": 0.25, "relevantMicroRecallDelta": 0.210526, "averageRetrievalStepsDelta": 0.25, "averageLatencyWorkUnitsDelta": 3.125, "unitAcceptedCostDelta": 0.008572 }, "failureSamples": { "fixedOneShot": [ { "queryId": "q01-atlas-supper", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q04-atlas-pto", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-people-leave" ] }, { "queryId": "q10-atlas-throttle", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-reliability-rate-limit" ] }, { "queryId": "q12-cinder-severity", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "cinder-reliability-incident" ] }, { "queryId": "q13-atlas-erasure", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [ "atlas-governance-customer-retention" ] }, { "queryId": "q14-boreal-former-worker", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q21-atlas-composite", "reasons": [ "answer-content-mismatch" ], "missedRelevant": [ "atlas-access-offboarding" ] }, { "queryId": "q23-cinder-p1-log-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q24-atlas-residency-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] } ], "adaptiveStateful": [ { "queryId": "q21-atlas-composite", "reasons": [ "answer-content-mismatch" ], "missedRelevant": [ "atlas-access-offboarding" ] }, { "queryId": "q23-cinder-p1-log-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] }, { "queryId": "q24-atlas-residency-gap", "reasons": [ "decision-type-mismatch" ], "missedRelevant": [] } ] } }