Proven (Established October 5, 2025 - GREAT-4A validation)
Intent classification systems require precise pattern matching to route natural language queries to appropriate handlers. Traditional approaches either use overly broad patterns (causing false positives) or overly narrow patterns (missing valid queries). The Intent Pattern Catalog pattern provides a systematic approach to defining, organizing, and maintaining regex patterns for intent classification with high precision and performance.
The Intent Pattern Catalog pattern organizes intent classification patterns into category-specific collections with standardized regex conventions, performance validation, and developer guidelines. The pattern ensures consistent pattern design, prevents category overlap, and maintains sub-millisecond classification performance.
Core concept:
# services/intent_service/pre_classifier.py
class PreClassifier:
"""Rule-based pre-classification for common patterns"""
# TEMPORAL patterns - time/date/schedule queries
TEMPORAL_PATTERNS = [
r"\bwhat day is it\b", # "What day is it?"
r"\bwhat'?s the date\b", # "What's the date?"
r"\bwhat time is it\b", # "What time is it?"
r"\bcurrent date\b", # "Current date"
r"\btoday'?s date\b", # "Today's date"
r"\bwhat'?s today\b", # "What's today?"
r"\bdate and time\b", # "Date and time"
]
# STATUS patterns - project/work status queries
STATUS_PATTERNS = [
r"\bwhat am i working on\b", # "What am I working on?"
r"\bwhat'?s my current project\b", # "What's my current project?"
r"\bmy projects\b", # "My projects"
r"\bcurrent work\b", # "Current work"
r"\bwhat'?s on my plate\b", # "What's on my plate?"
r"\bmy portfolio\b", # "My portfolio"
r"\bwhat'?s my status\b", # "What's my status?"
r"\bproject status\b", # "Project status"
]
# PRIORITY patterns - priority/focus queries
PRIORITY_PATTERNS = [
r"\bwhat'?s my top priority\b", # "What's my top priority?"
r"\bhighest priority\b", # "Highest priority"
r"\bmost important task\b", # "Most important task"
r"\bwhat should i do first\b", # "What should I do first?"
r"\bmy priorities\b", # "My priorities"
r"\btop priority\b", # "Top priority"
r"\bpriority one\b", # "Priority one"
]
@staticmethod
def pre_classify(message: str) -> Optional[Intent]:
"""Pre-classify message using rule-based patterns"""
clean_msg = message.strip().lower()
clean_for_matching = clean_msg.rstrip(string.punctuation + "!?.,;:😊🙂👋")
# Check TEMPORAL patterns
if PreClassifier._matches_patterns(clean_for_matching, PreClassifier.TEMPORAL_PATTERNS):
return Intent(
category=IntentCategory.TEMPORAL,
action="get_current_time",
confidence=1.0,
context={"original_message": message},
)
# Check STATUS patterns
if PreClassifier._matches_patterns(clean_for_matching, PreClassifier.STATUS_PATTERNS):
return Intent(
category=IntentCategory.STATUS,
action="get_project_status",
confidence=1.0,
context={"original_message": message},
)
# Check PRIORITY patterns
if PreClassifier._matches_patterns(clean_for_matching, PreClassifier.PRIORITY_PATTERNS):
return Intent(
category=IntentCategory.PRIORITY,
action="get_top_priority",
confidence=1.0,
context={"original_message": message},
)
return None
@staticmethod
def _matches_patterns(message: str, patterns: list) -> bool:
"""Check if message matches any of the given patterns using regex"""
for pattern in patterns:
if re.search(pattern, message):
return True
return False
# Pattern Design Guidelines
class IntentPatternConventions:
"""Standardized conventions for intent pattern design"""
@staticmethod
def create_pattern(base_phrase: str, variations: List[str] = None) -> str:
"""Create standardized regex pattern with conventions"""
# 1. Use word boundaries to prevent partial matches
# 2. Handle optional apostrophes with '?'
# 3. Keep case-insensitive (handled by preprocessing)
# Convert phrase to regex with word boundaries
pattern = rf"\b{base_phrase.lower()}\b"
# Handle common contractions
pattern = pattern.replace("what's", "what'?s")
pattern = pattern.replace("today's", "today'?s")
pattern = pattern.replace("what is", "what'?s")
return pattern
@staticmethod
def validate_pattern(pattern: str, test_queries: List[str],
expected_matches: List[bool]) -> Dict[str, Any]:
"""Validate pattern against test queries"""
results = []
for query, expected in zip(test_queries, expected_matches):
clean_query = query.strip().lower().rstrip("!?.,;:")
actual = bool(re.search(pattern, clean_query))
results.append({
"query": query,
"expected": expected,
"actual": actual,
"correct": actual == expected
})
accuracy = sum(1 for r in results if r["correct"]) / len(results)
return {
"pattern": pattern,
"accuracy": accuracy,
"results": results,
"passed": accuracy >= 0.9 # 90% accuracy threshold
}
# Performance validation for pattern catalog
class PatternPerformanceValidator:
"""Validate performance of intent pattern catalog"""
def __init__(self):
self.performance_targets = {
"max_time_ms": 100, # <100ms per classification
"min_confidence": 0.8, # >0.8 confidence for matches
"min_success_rate": 0.9 # >90% success rate
}
async def validate_category_performance(self, category_name: str,
test_queries: List[str]) -> Dict[str, Any]:
"""Validate performance for a specific category"""
from services.intent_service.classifier import classifier
times = []
confidences = []
successful_queries = []
for query in test_queries:
try:
start = time.perf_counter()
result = await classifier.classify(query)
elapsed = time.perf_counter() - start
times.append(elapsed * 1000) # Convert to ms
confidences.append(result.confidence)
successful_queries.append(query)
except Exception:
# Query failed - not counted in performance metrics
pass
if not times:
return {"error": "No successful classifications"}
metrics = {
"category": category_name,
"num_queries": len(test_queries),
"successful_queries": len(successful_queries),
"avg_time_ms": statistics.mean(times),
"median_time_ms": statistics.median(times),
"max_time_ms": max(times),
"min_time_ms": min(times),
"avg_confidence": statistics.mean(confidences),
"min_confidence": min(confidences),
"success_rate": len(successful_queries) / len(test_queries)
}
# Validate against targets
metrics["performance_validation"] = {
"time_ok": metrics["avg_time_ms"] < self.performance_targets["max_time_ms"],
"confidence_ok": metrics["avg_confidence"] >= self.performance_targets["min_confidence"],
"success_ok": metrics["success_rate"] >= self.performance_targets["min_success_rate"],
"overall_pass": all([
metrics["avg_time_ms"] < self.performance_targets["max_time_ms"],
metrics["avg_confidence"] >= self.performance_targets["min_confidence"],
metrics["success_rate"] >= self.performance_targets["min_success_rate"]
])
}
return metrics
# Category management for intent patterns
class IntentCategoryManager:
"""Manage intent categories and prevent overlap"""
def __init__(self):
self.category_definitions = {
"TEMPORAL": {
"description": "Time, date, schedule, and temporal reference queries",
"keywords": ["day", "date", "time", "today", "yesterday", "schedule"],
"actions": ["get_current_time"],
"examples": ["What day is it?", "What's today's date?"],
"handler": "canonical_handlers._handle_temporal_query"
},
"STATUS": {
"description": "Project/work status and current activity queries",
"keywords": ["working", "project", "status", "portfolio", "current"],
"actions": ["get_project_status"],
"examples": ["What am I working on?", "My projects"],
"handler": "canonical_handlers._handle_status_query"
},
"PRIORITY": {
"description": "Priority, focus, and importance queries",
"keywords": ["priority", "important", "focus", "first", "top"],
"actions": ["get_top_priority"],
"examples": ["What's my top priority?", "Most important task"],
"handler": "canonical_handlers._handle_priority_query"
}
}
def check_category_overlap(self, new_pattern: str, target_category: str) -> Dict[str, Any]:
"""Check if new pattern overlaps with existing categories"""
overlaps = []
for category, definition in self.category_definitions.items():
if category == target_category:
continue
# Check keyword overlap
pattern_words = set(re.findall(r'\w+', new_pattern.lower()))
category_keywords = set(definition["keywords"])
overlap_words = pattern_words.intersection(category_keywords)
if overlap_words:
overlaps.append({
"category": category,
"overlap_words": list(overlap_words),
"risk_level": "high" if len(overlap_words) > 1 else "medium"
})
return {
"has_overlaps": len(overlaps) > 0,
"overlaps": overlaps,
"recommendation": "Review pattern specificity" if overlaps else "Pattern is category-specific"
}
def suggest_pattern_improvements(self, pattern: str, category: str) -> List[str]:
"""Suggest improvements for pattern design"""
suggestions = []
# Check for word boundaries
if not pattern.startswith(r"\b") or not pattern.endswith(r"\b"):
suggestions.append("Add word boundaries (\\b) to prevent partial matches")
# Check for apostrophe handling
if "'" in pattern and "'?" not in pattern:
suggestions.append("Use '?' for optional apostrophes (what's → what'?s)")
# Check category-specific keywords
category_def = self.category_definitions.get(category, {})
category_keywords = category_def.get("keywords", [])
pattern_words = set(re.findall(r'\w+', pattern.lower()))
if not any(keyword in pattern_words for keyword in category_keywords):
suggestions.append(f"Consider including category keywords: {category_keywords}")
return suggestions
\b prevents partial matches (“priority” vs “prioritize”)'? for optional apostrophes (“what’s” vs “whats”)# Monitor pattern performance in production
class PatternPerformanceMonitor:
def __init__(self):
self.metrics = defaultdict(lambda: {
"total_matches": 0,
"avg_time_ms": 0.0,
"confidence_scores": []
})
def record_pattern_match(self, category: str, time_ms: float, confidence: float):
"""Record pattern match for monitoring"""
metrics = self.metrics[category]
metrics["total_matches"] += 1
# Update running average
current_avg = metrics["avg_time_ms"]
total_matches = metrics["total_matches"]
metrics["avg_time_ms"] = ((current_avg * (total_matches - 1)) + time_ms) / total_matches
# Track confidence scores
metrics["confidence_scores"].append(confidence)
# Keep only recent scores (last 100)
if len(metrics["confidence_scores"]) > 100:
metrics["confidence_scores"] = metrics["confidence_scores"][-100:]
Post-enhancement accuracy results from Phase 3 testing:
| Category | Accuracy | Status | Variants Tested | Notes |
|---|---|---|---|---|
| PRIORITY | 100.0% | ✅ Exceeds Target | 25 queries | Perfect classification |
| TEMPORAL | 96.7% | ✅ Meets Target | 30 queries | Personal calendar/schedule queries |
| STATUS | 96.7% | ✅ Meets Target | 30 queries | Personal work status queries |
| IDENTITY | 76.0% | ⚠️ Below Target | 25 queries | Capability queries sometimes → QUERY |
| GUIDANCE | 76.7% | ⚠️ Below Target | 30 queries | Advice sometimes → CONVERSATION/STRATEGY |
Overall canonical accuracy: 89.3% (126 correct / 141 total)
Core mission achieved: TEMPORAL, STATUS, and PRIORITY (the three categories with timeout issues) now exceed 95% accuracy target.
Strong canonical signals (high confidence):
Disambiguation rules working:
Remaining challenges:
\b causing partial matchesIncrease canonical query coverage from 24% to >80% through comprehensive pattern additions.
.* and \w+ for variation handlingTEMPORAL Additions (+10):
r"\bday of the week\b" - “What day of the week is it?”r"\bwhat.*yesterday\b" - “What did we accomplish yesterday?”r"\bagenda.*today\b" - “What’s on the agenda for today?”r"\blast time.*worked\b" - “When was the last time we worked on this?”r"\bhow long.*working\b" - “How long have we been working on this?”STATUS Additions (+6):
r"\bshow.*projects\b" - “Show me current projects”r"\bproject overview\b" - “Give me a project overview”r"\blist.*projects\b" - “List all projects”r"\bprojects.*working on\b" - “What projects are we working on?”PRIORITY Additions (+6):
r"\bmost important\b" - “What’s most important?”r"\bneeds.*attention\b" - “What needs my attention?”r"\bwhat should i focus on\b" - Moved from GUIDANCE for specificityr"\bwhat.*focus on\b" - “What should I focus on today?”Both edge cases are acceptable LLM fallbacks for context-heavy queries.
.*) improved match rates significantly\b) critical for precisionservices/intent_service/pre_classifier.pydev/2025/10/05/baseline_metrics.jsondev/2025/10/05/intent-category-usage-guide.mddev/2025/10/05/intent-pattern-catalog.mdPattern established: October 5, 2025 Status: Proven with validated performance metrics Total patterns: 44 patterns across 3 categories (Phase 3 expansion complete)
Last Updated: October 5, 2025 Epic: GREAT-4B - Universal Intent Enforcement GitHub Issue: #206
web/middleware/intent_enforcement.pyGET /api/admin/intent-monitoringKey Features:
request.state.intent_required = Truetests/intent/test_bypass_prevention.pypython scripts/check_intent_bypasses.pyGET /query in web/api/routes/knowledge_graph.py, a structured graph filter rather than NL); no other NL-like route foundservices/intent_service/cache.pyGET /api/admin/intent-cache-metricsPOST /api/admin/intent-cache-clearCache Strategy:
Input vs Output Flow:
User INPUT → Intent Classification (enforced here)
↓
Handler → Response Generation
↓
Piper OUTPUT → Personality Enhancement (separate concern)
What Requires Intent:
piper issues create --title "Bug")Pattern Coverage: 92% canonical queries (23/25 passing)
Performance Metrics:
Test Results:
Production Status: Ready for deployment
Phase 4 (Optional):
Phase 5 (Optional):
dev/2025/10/05/middleware-implementation.mddev/2025/10/05/cache-implementation.mddev/2025/10/05/intent-baseline-FINAL.mddev/2025/10/05/2025-10-05-1540-prog-code-log.md11053f77: Infrastructure validation (Phase 0/1)d1010afb: IntentEnforcementMiddleware (Phase 2)116d59fb: Intent caching implementation (Phase 3)Implementation completed: October 5, 2025, 6:15 PM Total duration: ~2.5 hours across 4 phases Quality: Exceeds all acceptance criteria