{ "schemaVersion": 2, "id": "analytics-is-evidence-not-authority", "path": "analytics-is-evidence-not-authority", "title": "Analytics is evidence, not authority", "shortTitle": "Analytics is evidence, not authority", "description": "A practical guide to using adoption, cost, and value analytics as evidence without mistaking a dashboard for an agent operating model.", "targetQuestion": "Why are adoption, cost, and ROI dashboards not enough to operate a Microsoft agent estate?", "directAnswer": "Analytics can show usage, cost, and modeled value. It cannot by itself establish complete inventory, ownership, capability, authority, execution, or current system state. Use it as evidence inside a seven-question operating review, then verify consequential outcomes in the system of record.", "directAnswerSourceIds": [ "microsoft-measurement-reporting", "microsoft-consumption-dashboard", "agent-roles-permissions" ], "scope": "This guide explains how to use public Microsoft adoption, cost, and value tools as evidence inside an agent operating model. It does not assess every Microsoft product or claim that Celest currently executes Microsoft lifecycle changes. Celest's Microsoft-facing Estate Review remains read-only.", "evidenceLevel": "Celest-authored operating model grounded in the stated scopes and limitations of public Microsoft tools plus one source-minimized Customer Zero observation.", "author": { "name": "Celest", "url": "https://celest.dev/" }, "published": "2026-07-29", "updated": "2026-07-29", "sections": [ { "id": "the-dashboard-is-doing-its-job", "heading": "The dashboard is doing its job", "blocks": [ { "type": "paragraph", "text": "Microsoft's public tools answer useful questions. The Microsoft AI Decision Framework and Agent Platform Advisor help people choose where work or an agent should start. Analytics Hub collects standalone tools for adoption, readiness, cost, and impact. CreditUsage attributes consumption, while ValueLens turns observed interactions into a modeled value story. These are useful answers. The mistake is asking them to prove a different kind of fact.", "sourceIds": [ "microsoft-ai-decision-framework", "agent-platform-advisor", "microsoft-measurement-reporting", "analytics-hub", "credit-usage", "value-lens" ] }, { "type": "callout", "label": "Try this first", "text": "Choose one agent that matters. Put the seven questions on one page, link each answer to its source, and mark every unsupported answer unknown." } ] }, { "id": "four-useful-tool-answers", "heading": "Four useful questions Microsoft tools can help answer", "blocks": [ { "type": "table", "caption": "Useful analytics questions and the boundary of each answer", "columns": [ "Question", "What that answer does not prove", "Useful answer" ], "rows": [ [ "What should we use?", "Which agents actually exist or how they are configured now", "A platform or work surface that fits the stated need" ], [ "Are people using it?", "A complete estate inventory, accountable ownership, or safe capability boundaries", "Observed interactions, reach, activity, or adoption patterns" ], [ "What did it cost?", "That work completed or produced the intended result", "Observed consumption, allocation, utilization, or chargeback" ], [ "Can we tell an ROI story?", "That the modeled benefit occurred for this task or that the current system state is correct", "A stated model that maps activity to time, value, or investment" ] ], "rowSourceIds": [ [ "microsoft-ai-decision-framework", "agent-platform-advisor" ], [ "microsoft-measurement-reporting", "analytics-hub", "ai-in-one" ], [ "microsoft-consumption-dashboard", "copilot-studio-billing", "credit-usage" ], [ "copilot-business-impact", "value-lens" ] ] }, { "type": "callout", "label": "Keep the boundary clear", "text": "Usage is not inventory. Activity is not ownership. Cost is not success. Modeled value is not fresh system-of-record verification." }, { "type": "paragraph", "text": "A boundary does not make the metric less useful. It makes the metric safe to combine with other evidence. AI-in-One, for example, says that Purview audit logs can illuminate Copilot and agent interactions but are not the sole source of truth for licensing or full-fidelity activity. That is good evidence discipline: name what a source can establish, then stop.", "sourceIds": [ "ai-in-one" ] }, { "type": "paragraph", "text": "Consumption and billing are separate too. Microsoft's Consumption Dashboard says its credit data is for reference and directs operators to the relevant billing system for official charges. Copilot Studio gives an even simpler example: a proactive greeting can count as a billed Copilot Credit when the user never replies, while analytics classifies that same session as unengaged. A billed event does not establish engagement, and neither fact establishes a successful business outcome.", "sourceIds": [ "microsoft-consumption-dashboard", "copilot-studio-billing" ] } ] }, { "id": "seven-operator-questions", "heading": "Seven questions complete the operating loop", "blocks": [ { "type": "paragraph", "text": "An operator needs more than a scorecard. For one important agent, answer these seven questions with named sources and preserve unknown answers as unknown." }, { "type": "paragraph", "text": "Use one path through the review: inventory → ownership → capability → next decision → authority → execution → fresh state." }, { "type": "table", "caption": "The seven-question agent operating review", "columns": [ "Question", "Do not substitute", "A useful answer identifies" ], "rows": [ [ "1. What exists?", "A list of agents that happened to record activity", "The agent records in scope, their source systems, collection time, and completeness" ], [ "2. Who owns it?", "The creator, last user, or busiest department", "The accountable owner, sponsor, and decision boundary" ], [ "3. What can it do?", "Only the tools observed in recent usage", "Its tools, connectors, identities, permissions, data reach, and current constraints" ], [ "4. What should happen next?", "A red metric that silently becomes a command", "A bounded finding or proposal with rationale, consequences, alternatives, and missing evidence" ], [ "5. Who can authorize it?", "Possession of a credential, admin role, or approval-looking UI", "The principal or policy that may decide this exact intent, scope, and time" ], [ "6. Did it actually happen?", "Spend, an accepted request, progress text, or a provider success label", "Correlated execution evidence, including failure, uncertainty, and partial results" ], [ "7. What does the system of record say now?", "The agent's narration of what it believes it changed", "A fresh readback of the intended postcondition from the authoritative business system" ] ] }, { "type": "callout", "label": "The operating rule", "text": "Every answer may use analytics as evidence. No answer may borrow the authority of the next question." }, { "type": "paragraph", "text": "Microsoft already provides useful pieces of this review in different places. The Microsoft 365 admin center describes a centralized inventory view and identifies ownerless agents. Its role guidance separately says that several roles may view agent information while only selected roles may perform governance actions such as approving requests or assigning ownership. The separation is the point: visibility, ownership evidence, and governance authority are related facts, not one permission.", "sourceIds": [ "agent-registry", "agent-roles-permissions" ] } ] }, { "id": "cost-was-real-success-was-not", "heading": "Cost was real. Success was not established", "blocks": [ { "type": "paragraph", "text": "In one reduced internal example—not Microsoft product documentation—Celest recorded about 602.41 Copilot Credits for an overall task. During a later create phase, no final tool result was tied to that phase. A separate read-only Dataverse query found zero intended lead rows and zero intended opportunity rows. The honest conclusion was narrow: the cost was final, the execution attempt was uncertain, and fresh readback verified no resulting rows at observation time." }, { "type": "paragraph", "text": "The example does not prove that cost data is unreliable. It proves that cost and outcome answer different questions. A useful cost observation should name the task, source, period, attribution, and whether the amount is final or still accruing. A useful outcome claim still needs execution evidence and, for consequential work, fresh state from the system that owns the result." }, { "type": "callout", "label": "A sentence worth keeping", "text": "Cost is evidence that resources were consumed. It is not evidence that the intended work succeeded." } ] }, { "id": "use-dashboards-as-sources", "heading": "Use dashboards as sources, not rivals", "blocks": [ { "type": "paragraph", "text": "An agent control plane should not rebuild every adoption chart or ROI model. It should retain the useful result with its source, observation time, scope, method, and limitations, then join that evidence to the same agent record used for ownership, capability, findings, and decisions." }, { "type": "list", "items": [ "Collect the smallest useful observation and record where it came from.", "Join it to an agent only when the identifiers and environment match with stated confidence.", "Keep unavailable, unknown, false, and zero as different states.", "Let a reviewed rule produce a finding; do not let a metric produce a mutation.", "Record the authority decision separately from the credential used at runtime.", "After execution, read the intended state again from the system that owns it." ] }, { "type": "paragraph", "text": "The resulting loop is simple to say even when the estate is complicated: observe, understand, decide, hand off, execute, verify, and retain a receipt. The dashboard remains valuable because its claim stays intact instead of being stretched into authority it never had." } ] }, { "id": "start-with-one-agent", "heading": "Start with one agent", "blocks": [ { "type": "paragraph", "text": "Choose one agent with meaningful activity, spend, risk, or business importance. Put the seven questions on one page. Link each answer to its source, mark every gap as unknown, and choose one next decision that a named person can make. If work follows, verify the result in the business system before calling it complete." }, { "type": "callout", "label": "A useful first review", "text": "One agent. Seven answers. One named decision. One fresh check before success." } ] }, { "id": "what-celest-proves-today", "heading": "What Celest proves today", "blocks": [ { "type": "paragraph", "text": "Celest can collect, join, and explain read-only evidence about an agent estate while keeping findings, proposed work, human decisions, execution observations, and receipts distinct. Today, the Microsoft-facing Estate Review does not perform Microsoft lifecycle changes or independently verify their resulting state in production." }, { "type": "paragraph", "text": "That boundary is why the seven questions matter now. They are useful before automated execution exists, and they remain necessary after execution becomes possible. More autonomy increases the value of evidence, authority, and verification; it does not collapse them." } ] } ], "limitations": [ "The public repositories cited here do not represent every Microsoft product capability, private preview, commercial service, or future roadmap item.", "An analytics source can be incomplete, delayed, modeled, or joined incorrectly; record its source health and method before using it in a decision.", "Modeled ROI is useful for planning when its assumptions are visible. It is not task-level outcome verification.", "The Customer Zero observation is one source-minimized internal experiment, not customer validation or a statistical claim about Cowork tasks.", "Celest's current Microsoft-facing Estate Review remains read-only; provider execution and fresh-state lifecycle reconciliation are not claimed as current commercial capabilities." ], "sources": [ { "id": "microsoft-ai-decision-framework", "title": "Microsoft AI Decision Framework", "publisher": "Microsoft", "url": "https://github.com/microsoft/Microsoft-AI-Decision-Framework", "accessed": "2026-07-29", "supports": "A public methodology for choosing among Microsoft AI technologies using business, experience, technology, lifecycle, and governance considerations." }, { "id": "agent-platform-advisor", "title": "Agent Platform Advisor", "publisher": "Microsoft", "url": "https://github.com/microsoft/AgentPlatformAdvisor", "accessed": "2026-07-29", "supports": "A static advisor that routes users among Microsoft work surfaces and recommends an agent-building platform from stated needs." }, { "id": "analytics-hub", "title": "Analytics Hub", "publisher": "Microsoft", "url": "https://github.com/microsoft/Analytics-Hub", "accessed": "2026-07-29", "supports": "A collection of standalone tools for Copilot adoption, readiness, usage, cost, and business-impact analysis across several source systems." }, { "id": "microsoft-measurement-reporting", "title": "Copilot Control System measurement and reporting", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/copilot/copilot-control-system/measurement-reporting", "accessed": "2026-07-29", "supports": "Microsoft reporting capabilities for understanding Copilot and agent adoption, usage trends, organizational impact, and value." }, { "id": "microsoft-consumption-dashboard", "title": "Consumption Dashboard", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/viva/insights/org-team-insights/ai-cost-dashboard", "accessed": "2026-07-29", "supports": "Credit-consumption monitoring across agents, services, teams, and users; the documentation labels dashboard credit data as reference-only and directs readers to billing systems for official charges." }, { "id": "copilot-studio-billing", "title": "FAQ for Copilot Studio billing and licensing", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-copilot-studio/faq-billing-licensing", "accessed": "2026-07-29", "supports": "Billing and engagement are distinct: a proactive greeting can be billed even when the user does not reply and analytics marks the conversation unengaged." }, { "id": "copilot-business-impact", "title": "Copilot Business Impact report", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/viva/insights/advanced/analyst/templates/copilot-business-impact", "accessed": "2026-07-29", "supports": "A report that relates Copilot usage to customer-supplied business outcome measures and documents omitted variables, statistical limitations, and estimated assisted value." }, { "id": "agent-registry", "title": "Agent Registry in Microsoft 365 admin center", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/manage/agent-registry?view=o365-worldwide", "accessed": "2026-07-29", "supports": "A centralized agent inventory and governance view that includes ownership gaps, platform, channel, source, risk, and management actions." }, { "id": "agent-roles-permissions", "title": "Agent management roles and permissions in Microsoft 365 admin center", "publisher": "Microsoft Learn", "url": "https://learn.microsoft.com/en-us/microsoft-365/admin/manage/agent-roles-perms?view=o365-worldwide", "accessed": "2026-07-29", "supports": "A role boundary that distinguishes viewing agent information from governance actions such as approving requests, assigning ownership, and changing configuration." }, { "id": "ai-in-one", "title": "AI-in-One Dashboard", "publisher": "Microsoft", "url": "https://github.com/microsoft/AI-in-One-Dashboard", "accessed": "2026-07-29", "supports": "A Power BI adoption dashboard whose documentation states its preprocessing requirements and warns that Purview audit logs are not the sole source of truth for licensing or full-fidelity activity." }, { "id": "credit-usage", "title": "Credit Usage and Chargebacks", "publisher": "Microsoft", "url": "https://github.com/microsoft/CreditUsage", "accessed": "2026-07-29", "supports": "A Power BI report that joins a Copilot credit-consumption export with Entra organization data to analyze utilization, attribution, chargeback, and capacity risk." }, { "id": "value-lens", "title": "ValueLens for Microsoft Copilot", "publisher": "Microsoft", "url": "https://github.com/microsoft/ValueLens-for-Microsoft-Copilot", "accessed": "2026-07-29", "supports": "An experimental analytics template that maps interactions to research-sourced task baselines, hours saved, assisted value, adoption, cost, and governance views while documenting source limitations." } ], "related": [ { "title": "The Agent Estate Review methodology", "url": "https://celest.dev/learn/agent-estate-review", "description": "Use a bounded read-only method to turn scattered evidence into an honest decision queue." }, { "title": "Evidence, authority, and receipts", "url": "https://celest.dev/learn/evidence-authority-and-receipts", "description": "See why evidence, approval, execution, verification, and receipts must remain separate artifacts." }, { "title": "Explore the fictional review journey", "url": "https://demo.celest.dev/reviews/manny-tool-permission?agent=manny", "description": "Walk through one explicitly illustrative signal-to-receipt review." } ], "publication": { "canonicalUrl": "https://celest.dev/learn/analytics-is-evidence-not-authority", "mcpUrl": "https://celest.dev/learn/analytics-is-evidence-not-authority/mcp", "sourceDigest": "sha256:29c1ed6735e440f7149f6ccdaf65acd8ada0c515f14a58452a91403192b56408" } }