{ "schemaVersion": 2, "id": "analytics-is-evidence-not-authority", "path": "analytics-is-evidence-not-authority", "title": "Analytics is evidence, not authority", "shortTitle": "Analytics is evidence, not authority", "description": "A practical guide to using adoption, cost, and value analytics as evidence without mistaking a dashboard for an agent operating model.", "targetQuestion": "Why are adoption, cost, and ROI dashboards not enough to operate a Microsoft agent estate?", "directAnswer": "Analytics can show usage, cost, and modeled value. It can't by itself establish complete inventory, ownership, capability, authority, execution, or current system state. Use it as evidence inside a seven-question operating review, then verify consequential outcomes in the system of record.", "directAnswerSourceIds": [ "microsoft-measurement-reporting", "microsoft-consumption-dashboard", "agent-roles-permissions" ], "scope": "This guide shows how to turn Microsoft adoption, cost, and value signals into useful evidence without mistaking them for ownership, authority, or outcomes. Agent Estate Review uses these sources as part of a wider read-only operating picture today.", "evidenceLevel": "Celest-authored operating model grounded in the stated scopes and limitations of public Microsoft tools plus one source-minimized Customer Zero observation.", "indexLabel": "Analytics", "author": { "name": "Celest", "url": "https://celest.dev/" }, "published": "2026-07-29", "updated": "2026-08-03", "sections": [ { "id": "the-dashboard-is-doing-its-job", "heading": "The dashboard is doing its job", "blocks": [ { "type": "paragraph", "text": "Microsoft's public tools answer useful questions. The Microsoft AI Decision Framework and Agent Platform Advisor help people choose where work or an agent should start. Analytics Hub collects standalone tools for adoption, readiness, cost, and impact. CreditUsage attributes consumption, while ValueLens turns observed interactions into a modeled value story. These are useful answers. The mistake is asking them to prove a different kind of fact.", "sourceIds": [ "microsoft-ai-decision-framework", "agent-platform-advisor", "microsoft-measurement-reporting", "analytics-hub", "credit-usage", "value-lens" ] }, { "type": "callout", "label": "Try this first", "text": "Choose one agent that matters. Put the seven questions on one page, link each answer to its source, and mark every unsupported answer unknown." } ] }, { "id": "four-useful-tool-answers", "heading": "Four useful questions Microsoft tools can help answer", "blocks": [ { "type": "table", "caption": "Useful analytics questions and the boundary of each answer", "columns": [ "Question", "What that answer doesn't prove", "Useful answer" ], "rows": [ [ "What should we use?", "Which agents actually exist or how they are configured now", "A platform or work surface that fits the stated need" ], [ "Are people using it?", "A complete estate inventory, accountable ownership, or safe capability boundaries", "Observed interactions, reach, activity, or adoption patterns" ], [ "What did it cost?", "That work completed or produced the intended result", "Observed consumption, allocation, utilization, or chargeback" ], [ "Can we tell an ROI story?", "That the modeled benefit occurred for this task or that the current system state is correct", "A stated model that maps activity to time, value, or investment" ] ], "rowSourceIds": [ [ "microsoft-ai-decision-framework", "agent-platform-advisor" ], [ "microsoft-measurement-reporting", "analytics-hub", "ai-in-one" ], [ "microsoft-consumption-dashboard", "copilot-studio-billing", "credit-usage" ], [ "copilot-business-impact", "value-lens" ] ] }, { "type": "callout", "label": "Keep the boundary clear", "text": "Usage is not inventory. Activity is not ownership. Cost is not success. Modeled value is not fresh system-of-record verification." }, { "type": "paragraph", "text": "A boundary doesn't make the metric less useful. It makes the metric safe to combine with other evidence. AI-in-One, for example, says that Purview audit logs can illuminate Copilot and agent interactions but are not the sole source of truth for licensing or full-fidelity activity. That is good evidence discipline: name what a source can establish, then stop.", "sourceIds": [ "ai-in-one" ] }, { "type": "paragraph", "text": "Consumption and billing are separate too. Microsoft's Consumption Dashboard says its credit data is for reference and directs operators to the relevant billing system for official charges. Copilot Studio gives an even simpler example: even when the user never replies, a proactive greeting can count as a billed Copilot Credit, while analytics classifies that same session as unengaged. A billed event doesn't establish engagement, and neither fact establishes a successful business outcome.", "sourceIds": [ "microsoft-consumption-dashboard", "copilot-studio-billing" ] } ] }, { "id": "seven-operator-questions", "heading": "Seven questions complete the operating loop", "blocks": [ { "type": "paragraph", "text": "An operator needs more than a scorecard. For one important agent, answer these seven questions with named sources and preserve unknown answers as unknown." }, { "type": "paragraph", "text": "Use one path through the review: inventory → ownership → capability → next decision → authority → execution → fresh state." }, { "type": "table", "caption": "The seven-question agent operating review", "columns": [ "Question", "Don't substitute", "A useful answer identifies" ], "rows": [ [ "1. What exists?", "A list of agents that happened to record activity", "The agent records in scope, their source systems, collection time, and completeness" ], [ "2. Who owns it?", "The creator, last user, or busiest department", "The accountable owner, sponsor, and decision boundary" ], [ "3. What can it do?", "Only the tools observed in recent usage", "Its tools, connectors, identities, permissions, data reach, and current constraints" ], [ "4. What should happen next?", "A red metric that silently becomes a command", "A bounded finding or proposal with rationale, consequences, alternatives, and missing evidence" ], [ "5. Who can authorize it?", "Possession of a credential, admin role, or approval-looking UI", "The principal or policy that may decide this exact intent, scope, and time" ], [ "6. Did it actually happen?", "Spend, an accepted request, progress text, or a provider success label", "Correlated execution evidence, including failure, uncertainty, and partial results" ], [ "7. What does the system of record say now?", "The agent's narration of what it believes it changed", "A fresh readback of the intended postcondition from the authoritative business system" ] ] }, { "type": "callout", "label": "The operating rule", "text": "Every answer may use analytics as evidence. No answer may borrow the authority of the next question." }, { "type": "paragraph", "text": "Microsoft already provides useful pieces of this review in different places. The Microsoft 365 admin center describes a centralized inventory view and identifies ownerless agents. Its role guidance separately says that several roles may view agent information while only selected roles may perform governance actions such as approving requests or assigning ownership. The separation is the point: visibility, ownership evidence, and governance authority are related facts, not one permission.", "sourceIds": [ "agent-registry", "agent-roles-permissions" ] } ] }, { "id": "cost-was-real-success-was-not", "heading": "Cost was real. Success was not established", "blocks": [ { "type": "paragraph", "text": "In one reduced internal example—not Microsoft product documentation—Celest recorded about 602.41 Copilot Credits for an overall task. During a later create phase, no final tool result was tied to that phase. A separate read-only Dataverse query found zero intended lead rows and zero intended opportunity rows. The honest conclusion was narrow: the cost was final, the execution attempt was uncertain, and fresh readback verified no resulting rows at observation time." }, { "type": "paragraph", "text": "The example doesn't prove that cost data is unreliable. It proves that cost and outcome answer different questions. A useful cost observation should name the task, source, period, attribution, and whether the amount is final or still accruing. A useful outcome claim still needs execution evidence and, for consequential work, fresh state from the system that owns the result." }, { "type": "callout", "label": "A sentence worth keeping", "text": "Cost is evidence that resources were consumed. It is not evidence that the intended work succeeded." } ] }, { "id": "use-dashboards-as-sources", "heading": "Use dashboards as sources, not rivals", "blocks": [ { "type": "paragraph", "text": "An agent control plane should not rebuild every adoption chart or ROI model. It should retain the useful result with its source, observation time, scope, method, and limitations, then join that evidence to the same agent record used for ownership, capability, findings, and decisions." }, { "type": "list", "items": [ "Collect the smallest useful observation and record where it came from.", "Only when the identifiers and environment match with stated confidence should you join it to an agent.", "Keep unavailable, unknown, false, and zero as different states.", "Let a reviewed rule produce a finding; don't let a metric produce a mutation.", "Record the authority decision separately from the credential used at runtime.", "After execution, read the intended state again from the system that owns it." ] }, { "type": "paragraph", "text": "Even when the estate is complicated, the resulting loop is simple to say: observe, understand, decide, hand off, execute, verify, and retain a receipt. The dashboard remains valuable because its claim stays intact instead of being stretched into authority it never had." } ] }, { "id": "start-with-one-agent", "heading": "Start with one agent", "blocks": [ { "type": "paragraph", "text": "Choose one agent with meaningful activity, spend, risk, or business importance. Put the seven questions on one page. Link each answer to its source, mark every gap as unknown, and choose one next decision that a named person can make. If work follows, verify the result in the business system before calling it complete." }, { "type": "callout", "label": "A useful first review", "text": "One agent. Seven answers. One named decision. One fresh check before success." } ] }, { "id": "what-celest-proves-today", "heading": "What Celest proves today", "blocks": [ { "type": "paragraph", "text": "Agent Estate Review collects, joins, and explains evidence about an agent estate while keeping findings, proposed work, human decisions, execution observations, and receipts distinct. The Microsoft-facing review stops at the read-only plan today; later gates describe the control model." }, { "type": "paragraph", "text": "That boundary is why the seven questions matter now. They are useful before automated execution exists, and they remain necessary after execution becomes possible. More autonomy increases the value of evidence, authority, and verification; it doesn't collapse them." } ] } ], "limitations": [ "The public repositories cited here don't represent every Microsoft product capability, private preview, commercial service, or future roadmap item.", "An analytics source can be incomplete, delayed, modeled, or joined incorrectly; record its source health and method before using it in a decision.", "When its assumptions are visible, modeled ROI is useful for planning. It is not task-level outcome verification.", "The Customer Zero observation is one source-minimized internal experiment, not customer validation or a statistical claim about Cowork tasks.", "Agent Estate Review stops at the read-only plan today; provider execution and fresh-state reconciliation remain later stages in the control model." ], "sources": [ { "id": "microsoft-ai-decision-framework", "title": "Microsoft AI Decision Framework", "publisher": "Microsoft", "url": "https://github.com/microsoft/Microsoft-AI-Decision-Framework", "accessed": "2026-07-29", "supports": "A public methodology for choosing among Microsoft AI technologies using business, experience, technology, lifecycle, and governance considerations." }, { "id": "agent-platform-advisor", "title": "Agent Platform Advisor", "publisher": "Microsoft", "url": "https://github.com/microsoft/AgentPlatformAdvisor", "accessed": "2026-07-29", "supports": "A static advisor that routes users among Microsoft work surfaces and recommends an agent-building platform from stated needs." }, { "id": "analytics-hub", "title": "Analytics Hub", "publisher": "Microsoft", "url": "https://github.com/microsoft/Analytics-Hub", "accessed": "2026-07-29", "supports": "A collection of standalone tools for Copilot adoption, readiness, usage, cost, and business-impact analysis across several source systems." }, { "id": "microsoft-measurement-reporting", "title": "Copilot Control System measurement and reporting", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/copilot/copilot-control-system/measurement-reporting", "accessed": "2026-07-29", "supports": "Microsoft reporting capabilities for understanding Copilot and agent adoption, usage trends, organizational impact, and value." }, { "id": "microsoft-consumption-dashboard", "title": "Consumption Dashboard", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/viva/insights/org-team-insights/ai-cost-dashboard", "accessed": "2026-07-29", "supports": "Credit-consumption monitoring across agents, services, teams, and users; the documentation labels dashboard credit data as reference-only and directs readers to billing systems for official charges." }, { "id": "copilot-studio-billing", "title": "FAQ for Copilot Studio billing and licensing", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-copilot-studio/faq-billing-licensing", "accessed": "2026-07-29", "supports": "Billing and engagement are distinct: even when the user doesn't reply and analytics marks the conversation unengaged, a proactive greeting can be billed." }, { "id": "copilot-business-impact", "title": "Copilot Business Impact report", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/viva/insights/advanced/analyst/templates/copilot-business-impact", "accessed": "2026-07-29", "supports": "A report that relates Copilot usage to customer-supplied business outcome measures and documents omitted variables, statistical limitations, and estimated assisted value." }, { "id": "agent-registry", "title": "Agent Registry in Microsoft 365 admin center", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/manage/agent-registry?view=o365-worldwide", "accessed": "2026-07-29", "supports": "A centralized agent inventory and governance view that includes ownership gaps, platform, channel, source, risk, and management actions." }, { "id": "agent-roles-permissions", "title": "Agent management roles and permissions in Microsoft 365 admin center", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/manage/agent-roles-perms?view=o365-worldwide", "accessed": "2026-07-29", "supports": "A role boundary that distinguishes viewing agent information from governance actions such as approving requests, assigning ownership, and changing configuration." }, { "id": "ai-in-one", "title": "AI-in-One Dashboard", "publisher": "Microsoft", "url": "https://github.com/microsoft/AI-in-One-Dashboard", "accessed": "2026-07-29", "supports": "A Power BI adoption dashboard whose documentation states its preprocessing requirements and warns that Purview audit logs are not the sole source of truth for licensing or full-fidelity activity." }, { "id": "credit-usage", "title": "Credit Usage and Chargebacks", "publisher": "Microsoft", "url": "https://github.com/microsoft/CreditUsage", "accessed": "2026-07-29", "supports": "A Power BI report that joins a Copilot credit-consumption export with Entra organization data to analyze utilization, attribution, chargeback, and capacity risk." }, { "id": "value-lens", "title": "ValueLens for Microsoft Copilot", "publisher": "Microsoft", "url": "https://github.com/microsoft/ValueLens-for-Microsoft-Copilot", "accessed": "2026-07-29", "supports": "An experimental analytics template that maps interactions to research-sourced task baselines, hours saved, assisted value, adoption, cost, and governance views while documenting source limitations." } ], "related": [ { "title": "The Agent Estate Review methodology", "url": "https://celest.dev/learn/agent-estate-review", "description": "See how Celest turns scattered agent evidence into an accountable decision queue." }, { "title": "Evidence, authority, and receipts", "url": "https://celest.dev/learn/evidence-authority-and-receipts", "description": "See why evidence, approval, execution, verification, and receipts must remain separate artifacts." }, { "title": "Launch the Estate Review preview", "url": "https://demo.celest.dev/reviews/manny-tool-permission?agent=manny", "description": "Follow a sample mission from signal to decision, handoff, and receipt." } ], "publication": { "canonicalUrl": "https://celest.dev/learn/analytics-is-evidence-not-authority", "mcpUrl": "https://celest.dev/learn/analytics-is-evidence-not-authority/mcp", "sourceDigest": "sha256:f9086a3dde21c5d42ea960156960248eded604203c1fa025dc00e8b321496b09" } }