Coverage for little_loops / issue_history / quality.py: 0%
196 statements
« prev ^ index » next coverage.py v7.12.0, created at 2026-03-18 16:18 -0500
1"""Issue history quality analysis: test gaps, rejections, manual patterns, config gaps."""
3from __future__ import annotations
5import json
6import re
7from pathlib import Path
8from typing import Any
10from little_loops.issue_history._utils import get_issue_content
11from little_loops.issue_history.models import (
12 CompletedIssue,
13 ConfigGap,
14 ConfigGapsAnalysis,
15 HotspotAnalysis,
16 ManualPattern,
17 ManualPatternAnalysis,
18 RejectionAnalysis,
19 RejectionMetrics,
20 TestGap,
21 TestGapAnalysis,
22)
23from little_loops.issue_history.parsing import _find_test_file, _parse_resolution_action
26def analyze_test_gaps(
27 issues: list[CompletedIssue],
28 hotspots: HotspotAnalysis,
29) -> TestGapAnalysis:
30 """Correlate bug occurrences with test coverage gaps.
32 Args:
33 issues: List of completed issues (unused, for API consistency)
34 hotspots: Pre-computed hotspot analysis
36 Returns:
37 TestGapAnalysis with test coverage gap information
38 """
39 # Build map of source files to bug info from hotspots
40 bug_files: dict[str, dict[str, Any]] = {}
42 for hotspot in hotspots.file_hotspots:
43 bug_count = hotspot.issue_types.get("BUG", 0)
44 if bug_count > 0:
45 # Filter to only BUG issue IDs
46 bug_ids = [iid for iid in hotspot.issue_ids if iid.startswith("BUG-")]
47 bug_files[hotspot.path] = {
48 "bug_count": bug_count,
49 "bug_ids": bug_ids,
50 }
52 if not bug_files:
53 return TestGapAnalysis()
55 # Analyze test coverage for each file with bugs
56 gaps: list[TestGap] = []
57 files_with_tests: list[int] = [] # bug counts
58 files_without_tests: list[int] = [] # bug counts
60 for source_file, data in bug_files.items():
61 bug_count = data["bug_count"]
62 bug_ids = data["bug_ids"]
64 test_file = _find_test_file(source_file)
65 has_test = test_file is not None
67 # Calculate gap score: higher = more urgent to add tests
68 # Files without tests get amplified scores
69 if has_test:
70 gap_score = bug_count * 1.0
71 files_with_tests.append(bug_count)
72 else:
73 gap_score = bug_count * 10.0 # Amplify untested files
74 files_without_tests.append(bug_count)
76 # Determine priority based on bug count and test presence
77 if not has_test and bug_count >= 5:
78 priority = "critical"
79 elif not has_test and bug_count >= 3:
80 priority = "high"
81 elif not has_test or bug_count >= 4:
82 priority = "medium"
83 else:
84 priority = "low"
86 gaps.append(
87 TestGap(
88 source_file=source_file,
89 bug_count=bug_count,
90 bug_ids=bug_ids,
91 has_test_file=has_test,
92 test_file_path=test_file,
93 gap_score=gap_score,
94 priority=priority,
95 )
96 )
98 # Sort by gap score descending (highest priority first)
99 gaps.sort(key=lambda g: (-g.gap_score, -g.bug_count))
101 # Calculate averages for correlation
102 avg_with_tests = sum(files_with_tests) / len(files_with_tests) if files_with_tests else 0.0
103 avg_without_tests = (
104 sum(files_without_tests) / len(files_without_tests) if files_without_tests else 0.0
105 )
107 # Identify untested bug magnets (from hotspot analysis)
108 untested_magnets = [h.path for h in hotspots.bug_magnets if _find_test_file(h.path) is None]
110 # Priority test targets: untested files sorted by bug count
111 priority_targets = [g.source_file for g in gaps if not g.has_test_file]
113 return TestGapAnalysis(
114 gaps=gaps[:15], # Top 15
115 untested_bug_magnets=untested_magnets,
116 files_with_tests_avg_bugs=avg_with_tests,
117 files_without_tests_avg_bugs=avg_without_tests,
118 priority_test_targets=priority_targets[:10],
119 )
122def analyze_rejection_rates(
123 issues: list[CompletedIssue],
124 contents: dict[Path, str] | None = None,
125) -> RejectionAnalysis:
126 """Analyze rejection and invalid closure patterns.
128 Args:
129 issues: List of completed issues
130 contents: Pre-loaded issue file contents (path -> content)
132 Returns:
133 RejectionAnalysis with overall and grouped metrics
134 """
135 if not issues:
136 return RejectionAnalysis()
138 # Count by category
139 overall = RejectionMetrics()
140 by_type: dict[str, RejectionMetrics] = {}
141 by_month: dict[str, RejectionMetrics] = {}
142 reason_counts: dict[str, int] = {}
144 for issue in issues:
145 content = get_issue_content(issue, contents)
146 if content is None:
147 continue
149 category = _parse_resolution_action(content)
150 overall.total_closed += 1
152 # Update overall counts
153 if category == "completed":
154 overall.completed_count += 1
155 elif category == "rejected":
156 overall.rejected_count += 1
157 elif category == "invalid":
158 overall.invalid_count += 1
159 elif category == "duplicate":
160 overall.duplicate_count += 1
161 elif category == "deferred":
162 overall.deferred_count += 1
164 # By type
165 if issue.issue_type not in by_type:
166 by_type[issue.issue_type] = RejectionMetrics()
167 type_metrics = by_type[issue.issue_type]
168 type_metrics.total_closed += 1
169 if category == "rejected":
170 type_metrics.rejected_count += 1
171 elif category == "invalid":
172 type_metrics.invalid_count += 1
173 elif category == "duplicate":
174 type_metrics.duplicate_count += 1
175 elif category == "deferred":
176 type_metrics.deferred_count += 1
177 elif category == "completed":
178 type_metrics.completed_count += 1
180 # By month
181 if issue.completed_date:
182 month_key = issue.completed_date.strftime("%Y-%m")
183 if month_key not in by_month:
184 by_month[month_key] = RejectionMetrics()
185 month_metrics = by_month[month_key]
186 month_metrics.total_closed += 1
187 if category == "rejected":
188 month_metrics.rejected_count += 1
189 elif category == "invalid":
190 month_metrics.invalid_count += 1
191 elif category == "duplicate":
192 month_metrics.duplicate_count += 1
193 elif category == "deferred":
194 month_metrics.deferred_count += 1
195 elif category == "completed":
196 month_metrics.completed_count += 1
198 # Extract reason for rejection/invalid
199 if category in ("rejected", "invalid", "duplicate", "deferred"):
200 reason_match = re.search(r"\*\*Reason\*\*:\s*(.+?)(?:\n|$)", content)
201 if reason_match:
202 reason = reason_match.group(1).strip()
203 reason_counts[reason] = reason_counts.get(reason, 0) + 1
205 # Calculate trend from monthly data
206 sorted_months = sorted(by_month.keys())
207 if len(sorted_months) >= 3:
208 recent = sorted_months[-3:]
209 rates = [by_month[m].rejection_rate + by_month[m].invalid_rate for m in recent]
210 if rates[-1] < rates[0] * 0.8:
211 trend = "improving"
212 elif rates[-1] > rates[0] * 1.2:
213 trend = "degrading"
214 else:
215 trend = "stable"
216 else:
217 trend = "stable"
219 # Sort reasons by count
220 common_reasons = sorted(reason_counts.items(), key=lambda x: -x[1])[:10]
222 return RejectionAnalysis(
223 overall=overall,
224 by_type=by_type,
225 by_month=by_month,
226 common_reasons=common_reasons,
227 trend=trend,
228 )
231# Pattern definitions for manual activity detection
232_MANUAL_PATTERNS: dict[str, dict[str, Any]] = {
233 "test": {
234 "patterns": [
235 r"(?:pytest|python -m pytest|npm test|yarn test|jest|cargo test|go test)",
236 r"(?:python -m unittest|nosetests|tox)",
237 ],
238 "description": "Test execution after code changes",
239 "suggestion": "Add post-edit hook for automatic test runs",
240 "complexity": "trivial",
241 },
242 "lint": {
243 "patterns": [
244 r"(?:ruff check|ruff format|black|isort|flake8|pylint)",
245 r"(?:eslint|prettier|tslint)",
246 ],
247 "description": "Lint/format fixes after implementation",
248 "suggestion": "Add pre-commit hook for auto-formatting",
249 "complexity": "simple",
250 },
251 "type_check": {
252 "patterns": [
253 r"(?:mypy|pyright|python -m mypy)",
254 r"(?:tsc|npx tsc)",
255 ],
256 "description": "Type checking during development",
257 "suggestion": "Add mypy to pre-commit or post-edit hook",
258 "complexity": "simple",
259 },
260 "build": {
261 "patterns": [
262 r"(?:npm run build|yarn build|make|cargo build|go build)",
263 r"(?:python -m build|pip install -e)",
264 ],
265 "description": "Build steps during implementation",
266 "suggestion": "Add build verification to test suite or CI",
267 "complexity": "moderate",
268 },
269 "git": {
270 "patterns": [
271 r"git (?:add|commit|push|pull|checkout|branch)",
272 ],
273 "description": "Git operations during issue resolution",
274 "suggestion": "Use /ll:commit skill for standardized commits",
275 "complexity": "trivial",
276 },
277}
280def detect_manual_patterns(
281 issues: list[CompletedIssue],
282 contents: dict[Path, str] | None = None,
283) -> ManualPatternAnalysis:
284 """Detect recurring manual activities that could be automated.
286 Args:
287 issues: List of completed issues
288 contents: Pre-loaded issue file contents (path -> content)
290 Returns:
291 ManualPatternAnalysis with detected patterns
292 """
293 if not issues:
294 return ManualPatternAnalysis()
296 # Track pattern occurrences
297 pattern_data: dict[str, dict[str, Any]] = {}
299 for pattern_type, config in _MANUAL_PATTERNS.items():
300 pattern_data[pattern_type] = {
301 "count": 0,
302 "issues": [],
303 "commands": [],
304 "config": config,
305 }
307 # Scan issue content for patterns
308 for issue in issues:
309 content = get_issue_content(issue, contents)
310 if content is None:
311 continue
313 for pattern_type, config in _MANUAL_PATTERNS.items():
314 for pattern in config["patterns"]:
315 matches = re.findall(pattern, content, re.IGNORECASE)
316 if matches:
317 data = pattern_data[pattern_type]
318 data["count"] += len(matches)
319 if issue.issue_id not in data["issues"]:
320 data["issues"].append(issue.issue_id)
321 # Store unique command examples
322 for match in matches:
323 if match not in data["commands"]:
324 data["commands"].append(match)
326 # Build ManualPattern objects
327 patterns: list[ManualPattern] = []
328 total_interventions = 0
329 automatable = 0
331 for pattern_type, data in pattern_data.items():
332 if data["count"] > 0:
333 config = data["config"]
334 pattern = ManualPattern(
335 pattern_type=pattern_type,
336 pattern_description=config["description"],
337 occurrence_count=data["count"],
338 affected_issues=data["issues"],
339 example_commands=data["commands"][:5],
340 suggested_automation=config["suggestion"],
341 automation_complexity=config["complexity"],
342 )
343 patterns.append(pattern)
344 total_interventions += data["count"]
345 automatable += data["count"]
347 # Sort by occurrence count descending
348 patterns.sort(key=lambda p: -p.occurrence_count)
350 # Build automation suggestions
351 suggestions = [p.suggested_automation for p in patterns if p.occurrence_count >= 2]
353 return ManualPatternAnalysis(
354 patterns=patterns,
355 total_manual_interventions=total_interventions,
356 automatable_count=automatable,
357 automation_suggestions=suggestions[:10],
358 )
361# Mapping from manual pattern types to configuration solutions
362_PATTERN_TO_CONFIG: dict[str, dict[str, Any]] = {
363 "test": {
364 "hook_event": "PostToolUse",
365 "description": "Automatic test execution after code changes",
366 "suggested_config": """hooks/hooks.json:
367 "PostToolUse": [{
368 "matcher": "Edit|Write",
369 "hooks": [{
370 "type": "command",
371 "command": "pytest tests/ -x -q",
372 "timeout": 30000
373 }]
374 }]""",
375 },
376 "lint": {
377 "hook_event": "PreToolUse",
378 "description": "Automatic formatting before file writes",
379 "suggested_config": """hooks/hooks.json:
380 "PreToolUse": [{
381 "matcher": "Write|Edit",
382 "hooks": [{
383 "type": "command",
384 "command": "ruff format --check .",
385 "timeout": 10000
386 }]
387 }]""",
388 },
389 "type_check": {
390 "hook_event": "PostToolUse",
391 "description": "Type checking after code modifications",
392 "suggested_config": """hooks/hooks.json:
393 "PostToolUse": [{
394 "matcher": "Edit|Write",
395 "hooks": [{
396 "type": "command",
397 "command": "mypy --fast .",
398 "timeout": 30000
399 }]
400 }]""",
401 },
402 "build": {
403 "hook_event": "PostToolUse",
404 "description": "Build verification after changes",
405 "suggested_config": """hooks/hooks.json:
406 "PostToolUse": [{
407 "matcher": "Edit|Write",
408 "hooks": [{
409 "type": "command",
410 "command": "npm run build",
411 "timeout": 60000
412 }]
413 }]""",
414 },
415}
418def detect_config_gaps(
419 manual_pattern_analysis: ManualPatternAnalysis,
420 project_root: Path | None = None,
421) -> ConfigGapsAnalysis:
422 """Detect configuration gaps based on manual pattern analysis.
424 Args:
425 manual_pattern_analysis: Results from detect_manual_patterns()
426 project_root: Project root directory (defaults to cwd)
428 Returns:
429 ConfigGapsAnalysis with identified gaps and coverage metrics
430 """
431 if project_root is None:
432 project_root = Path.cwd()
434 # Discover current configuration
435 current_hooks: list[str] = []
436 current_skills: list[str] = []
437 current_agents: list[str] = []
439 # Load hooks configuration
440 hooks_file = project_root / "hooks" / "hooks.json"
441 if hooks_file.exists():
442 try:
443 with open(hooks_file, encoding="utf-8") as f:
444 hooks_data = json.load(f)
445 current_hooks = list(hooks_data.get("hooks", {}).keys())
446 except Exception:
447 pass
449 # Scan for agents
450 agents_dir = project_root / "agents"
451 if agents_dir.is_dir():
452 for agent_file in agents_dir.glob("*.md"):
453 current_agents.append(agent_file.stem)
455 # Scan for skills
456 skills_dir = project_root / "skills"
457 if skills_dir.is_dir():
458 for skill_dir in skills_dir.iterdir():
459 if skill_dir.is_dir() and (skill_dir / "SKILL.md").exists():
460 current_skills.append(skill_dir.name)
462 # Identify gaps from manual patterns
463 gaps: list[ConfigGap] = []
464 covered_patterns = 0
465 recognized_patterns = 0
467 for pattern in manual_pattern_analysis.patterns:
468 config_mapping = _PATTERN_TO_CONFIG.get(pattern.pattern_type)
469 if not config_mapping:
470 continue
472 recognized_patterns += 1
473 hook_event = config_mapping["hook_event"]
475 # Check if hook event is already configured
476 if hook_event in current_hooks:
477 covered_patterns += 1
478 continue
480 # Determine priority based on occurrence count
481 if pattern.occurrence_count >= 10:
482 priority = "high"
483 elif pattern.occurrence_count >= 5:
484 priority = "medium"
485 else:
486 priority = "low"
488 gap = ConfigGap(
489 gap_type="hook",
490 description=config_mapping["description"],
491 evidence=pattern.affected_issues,
492 suggested_config=config_mapping["suggested_config"],
493 priority=priority,
494 pattern_type=pattern.pattern_type,
495 )
496 gaps.append(gap)
498 # Calculate coverage score based on recognized patterns only
499 coverage_score = covered_patterns / recognized_patterns if recognized_patterns > 0 else 1.0
501 # Sort gaps by priority (high first)
502 priority_order = {"high": 0, "medium": 1, "low": 2}
503 gaps.sort(key=lambda g: priority_order.get(g.priority, 3))
505 return ConfigGapsAnalysis(
506 gaps=gaps,
507 current_hooks=current_hooks,
508 current_skills=current_skills,
509 current_agents=current_agents,
510 coverage_score=coverage_score,
511 )