Pilot Metrics

What you measure in the pilot is what the rest of the organization will believe about the AI. Measure vanity metrics and the organization will question whether the results are real. Measure workflow metrics — specific, measurable changes in how work gets done — and the results are defensible.

{
  "pilotMetrics": {
    "adoptionRate": {
      "definition": "Percentage of pilot team using the tool for its intended purpose at least 3x per week",
      "target": ">= 75%",
      "measurement": "Tool usage logs, task completion records",
      "note": "Distinguish intended use from incidental logins"
    },
    "taskCompletionImprovement": {
      "definition": "Change in time to complete the target workflow vs. pre-pilot baseline",
      "target": ">= 20% reduction",
      "measurement": "Timed task completion, before and after sampling",
      "note": "Use the same workflow, same complexity, different time period"
    },
    "timeSavings": {
      "definition": "Hours per person per week recovered from the targeted workflow",
      "target": ">= 2 hours/person/week",
      "measurement": "Time tracking, self-report with manager validation",
      "note": "Track what recovered time is used for — redeployed time is more compelling than saved time"
    },
    "errorReduction": {
      "definition": "Change in error rate or rework rate for the target output",
      "target": ">= 15% reduction",
      "measurement": "Output quality review, rework ticket counts, manager assessment",
      "note": "Quality improvement often matters more to skeptics than speed improvement"
    },
    "userSentiment": {
      "definition": "Do pilot team members want to continue using the tool after the pilot ends?",
      "target": ">= 80% affirmative",
      "measurement": "Anonymous survey, end-of-pilot interviews",
      "note": "Net Promoter framing: would you recommend this to a colleague? Score separately from usage intent"
    },
    "replicationReadiness": {
      "definition": "Can a second team reproduce the results using only the pilot documentation?",
      "target": "Yes — documented and validated",
      "measurement": "Second-team validation exercise",
      "note": "This is the gate metric. If replication fails, the pilot is not complete."
    }
  }
}