Coverage for little_loops/issue_history/quality.py: 0%

1"""Issue history quality analysis: test gaps, rejections, manual patterns, config gaps."""

2

3 from __future__ import annotations

4

5 import json

6 import re

7 from pathlib import Path

8 from typing import Any

9

10 from little_loops.issue_history._utils import get_issue_content

11 from little_loops.issue_history.models import (

12 CompletedIssue,

13 ConfigGap,

14 ConfigGapsAnalysis,

15 HotspotAnalysis,

16 ManualPattern,

17 ManualPatternAnalysis,

18 RejectionAnalysis,

19 RejectionMetrics,

20 TestGap,

21 TestGapAnalysis,

22)

23 from little_loops.issue_history.parsing import _find_test_file, _parse_resolution_action

24

25

26 def analyze_test_gaps(

27 issues: list[CompletedIssue],

28 hotspots: HotspotAnalysis,

29) -> TestGapAnalysis:

30 """Correlate bug occurrences with test coverage gaps.

31

32 Args:

33 issues: List of completed issues (unused, for API consistency)

34 hotspots: Pre-computed hotspot analysis

35

36 Returns:

37 TestGapAnalysis with test coverage gap information

38 """

39 # Build map of source files to bug info from hotspots

40 bug_files: dict[str, dict[str, Any]] = {}

41

42 for hotspot in hotspots.file_hotspots:

43 bug_count = hotspot.issue_types.get("BUG", 0)

44 if bug_count > 0:

45 # Filter to only BUG issue IDs

46 bug_ids = [iid for iid in hotspot.issue_ids if iid.startswith("BUG-")]

47 bug_files[hotspot.path] = {

48 "bug_count": bug_count,

49 "bug_ids": bug_ids,

50 }

51

52 if not bug_files:

53 return TestGapAnalysis()

54

55 # Analyze test coverage for each file with bugs

56 gaps: list[TestGap] = []

57 files_with_tests: list[int] = [] # bug counts

58 files_without_tests: list[int] = [] # bug counts

59

60 for source_file, data in bug_files.items():

61 bug_count = data["bug_count"]

62 bug_ids = data["bug_ids"]

63

64 test_file = _find_test_file(source_file)

65 has_test = test_file is not None

66

67 # Calculate gap score: higher = more urgent to add tests

68 # Files without tests get amplified scores

69 if has_test:

70 gap_score = bug_count * 1.0

71 files_with_tests.append(bug_count)

72 else:

73 gap_score = bug_count * 10.0 # Amplify untested files

74 files_without_tests.append(bug_count)

75

76 # Determine priority based on bug count and test presence

77 if not has_test and bug_count >= 5:

78 priority = "critical"

79 elif not has_test and bug_count >= 3:

80 priority = "high"

81 elif not has_test or bug_count >= 4:

82 priority = "medium"

83 else:

84 priority = "low"

85

86 gaps.append(

87 TestGap(

88 source_file=source_file,

89 bug_count=bug_count,

90 bug_ids=bug_ids,

91 has_test_file=has_test,

92 test_file_path=test_file,

93 gap_score=gap_score,

94 priority=priority,

95 )

96 )

97

98 # Sort by gap score descending (highest priority first)

99 gaps.sort(key=lambda g: (-g.gap_score, -g.bug_count))

100

101 # Calculate averages for correlation

102 avg_with_tests = sum(files_with_tests) / len(files_with_tests) if files_with_tests else 0.0

103 avg_without_tests = (

104 sum(files_without_tests) / len(files_without_tests) if files_without_tests else 0.0

105 )

106

107 # Identify untested bug magnets (from hotspot analysis)

108 untested_magnets = [h.path for h in hotspots.bug_magnets if _find_test_file(h.path) is None]

109

110 # Priority test targets: untested files sorted by bug count

111 priority_targets = [g.source_file for g in gaps if not g.has_test_file]

112

113 return TestGapAnalysis(

114 gaps=gaps[:15], # Top 15

115 untested_bug_magnets=untested_magnets,

116 files_with_tests_avg_bugs=avg_with_tests,

117 files_without_tests_avg_bugs=avg_without_tests,

118 priority_test_targets=priority_targets[:10],

119 )

120

121

122 def analyze_rejection_rates(

123 issues: list[CompletedIssue],

124 contents: dict[Path, str] | None = None,

125) -> RejectionAnalysis:

126 """Analyze rejection and invalid closure patterns.

127

128 Args:

129 issues: List of completed issues

130 contents: Pre-loaded issue file contents (path -> content)

131

132 Returns:

133 RejectionAnalysis with overall and grouped metrics

134 """

135 if not issues:

136 return RejectionAnalysis()

137

138 # Count by category

139 overall = RejectionMetrics()

140 by_type: dict[str, RejectionMetrics] = {}

141 by_month: dict[str, RejectionMetrics] = {}

142 reason_counts: dict[str, int] = {}

143

144 for issue in issues:

145 content = get_issue_content(issue, contents)

146 if content is None:

147 continue

148

149 category = _parse_resolution_action(content)

150 overall.total_closed += 1

151

152 # Update overall counts

153 if category == "completed":

154 overall.completed_count += 1

155 elif category == "rejected":

156 overall.rejected_count += 1

157 elif category == "invalid":

158 overall.invalid_count += 1

159 elif category == "duplicate":

160 overall.duplicate_count += 1

161 elif category == "deferred":

162 overall.deferred_count += 1

163

164 # By type

165 if issue.issue_type not in by_type:

166 by_type[issue.issue_type] = RejectionMetrics()

167 type_metrics = by_type[issue.issue_type]

168 type_metrics.total_closed += 1

169 if category == "rejected":

170 type_metrics.rejected_count += 1

171 elif category == "invalid":

172 type_metrics.invalid_count += 1

173 elif category == "duplicate":

174 type_metrics.duplicate_count += 1

175 elif category == "deferred":

176 type_metrics.deferred_count += 1

177 elif category == "completed":

178 type_metrics.completed_count += 1

179

180 # By month

181 if issue.completed_date:

182 month_key = issue.completed_date.strftime("%Y-%m")

183 if month_key not in by_month:

184 by_month[month_key] = RejectionMetrics()

185 month_metrics = by_month[month_key]

186 month_metrics.total_closed += 1

187 if category == "rejected":

188 month_metrics.rejected_count += 1

189 elif category == "invalid":

190 month_metrics.invalid_count += 1

191 elif category == "duplicate":

192 month_metrics.duplicate_count += 1

193 elif category == "deferred":

194 month_metrics.deferred_count += 1

195 elif category == "completed":

196 month_metrics.completed_count += 1

197

198 # Extract reason for rejection/invalid

199 if category in ("rejected", "invalid", "duplicate", "deferred"):

200 reason_match = re.search(r"\*\*Reason\*\*:\s*(.+?)(?:\n|$)", content)

201 if reason_match:

202 reason = reason_match.group(1).strip()

203 reason_counts[reason] = reason_counts.get(reason, 0) + 1

204

205 # Calculate trend from monthly data

206 sorted_months = sorted(by_month.keys())

207 if len(sorted_months) >= 3:

208 recent = sorted_months[-3:]

209 rates = [by_month[m].rejection_rate + by_month[m].invalid_rate for m in recent]

210 if rates[-1] < rates[0] * 0.8:

211 trend = "improving"

212 elif rates[-1] > rates[0] * 1.2:

213 trend = "degrading"

214 else:

215 trend = "stable"

216 else:

217 trend = "stable"

218

219 # Sort reasons by count

220 common_reasons = sorted(reason_counts.items(), key=lambda x: -x[1])[:10]

221

222 return RejectionAnalysis(

223 overall=overall,

224 by_type=by_type,

225 by_month=by_month,

226 common_reasons=common_reasons,

227 trend=trend,

228 )

229

230

231# Pattern definitions for manual activity detection

232 _MANUAL_PATTERNS: dict[str, dict[str, Any]] = {

233 "test": {

234 "patterns": [

235 r"(?:pytest|python -m pytest|npm test|yarn test|jest|cargo test|go test)",

236 r"(?:python -m unittest|nosetests|tox)",

237 ],

238 "description": "Test execution after code changes",

239 "suggestion": "Add post-edit hook for automatic test runs",

240 "complexity": "trivial",

241 },

242 "lint": {

243 "patterns": [

244 r"(?:ruff check|ruff format|black|isort|flake8|pylint)",

245 r"(?:eslint|prettier|tslint)",

246 ],

247 "description": "Lint/format fixes after implementation",

248 "suggestion": "Add pre-commit hook for auto-formatting",

249 "complexity": "simple",

250 },

251 "type_check": {

252 "patterns": [

253 r"(?:mypy|pyright|python -m mypy)",

254 r"(?:tsc|npx tsc)",

255 ],

256 "description": "Type checking during development",

257 "suggestion": "Add mypy to pre-commit or post-edit hook",

258 "complexity": "simple",

259 },

260 "build": {

261 "patterns": [

262 r"(?:npm run build|yarn build|make|cargo build|go build)",

263 r"(?:python -m build|pip install -e)",

264 ],

265 "description": "Build steps during implementation",

266 "suggestion": "Add build verification to test suite or CI",

267 "complexity": "moderate",

268 },

269 "git": {

270 "patterns": [

271 r"git (?:add|commit|push|pull|checkout|branch)",

272 ],

273 "description": "Git operations during issue resolution",

274 "suggestion": "Use /ll:commit skill for standardized commits",

275 "complexity": "trivial",

276 },

277}

278

279

280 def detect_manual_patterns(

281 issues: list[CompletedIssue],

282 contents: dict[Path, str] | None = None,

283) -> ManualPatternAnalysis:

284 """Detect recurring manual activities that could be automated.

285

286 Args:

287 issues: List of completed issues

288 contents: Pre-loaded issue file contents (path -> content)

289

290 Returns:

291 ManualPatternAnalysis with detected patterns

292 """

293 if not issues:

294 return ManualPatternAnalysis()

295

296 # Track pattern occurrences

297 pattern_data: dict[str, dict[str, Any]] = {}

298

299 for pattern_type, config in _MANUAL_PATTERNS.items():

300 pattern_data[pattern_type] = {

301 "count": 0,

302 "issues": [],

303 "commands": [],

304 "config": config,

305 }

306

307 # Scan issue content for patterns

308 for issue in issues:

309 content = get_issue_content(issue, contents)

310 if content is None:

311 continue

312

313 for pattern_type, config in _MANUAL_PATTERNS.items():

314 for pattern in config["patterns"]:

315 matches = re.findall(pattern, content, re.IGNORECASE)

316 if matches:

317 data = pattern_data[pattern_type]

318 data["count"] += len(matches)

319 if issue.issue_id not in data["issues"]:

320 data["issues"].append(issue.issue_id)

321 # Store unique command examples

322 for match in matches:

323 if match not in data["commands"]:

324 data["commands"].append(match)

325

326 # Build ManualPattern objects

327 patterns: list[ManualPattern] = []

328 total_interventions = 0

329 automatable = 0

330

331 for pattern_type, data in pattern_data.items():

332 if data["count"] > 0:

333 config = data["config"]

334 pattern = ManualPattern(

335 pattern_type=pattern_type,

336 pattern_description=config["description"],

337 occurrence_count=data["count"],

338 affected_issues=data["issues"],

339 example_commands=data["commands"][:5],

340 suggested_automation=config["suggestion"],

341 automation_complexity=config["complexity"],

342 )

343 patterns.append(pattern)

344 total_interventions += data["count"]

345 automatable += data["count"]

346

347 # Sort by occurrence count descending

348 patterns.sort(key=lambda p: -p.occurrence_count)

349

350 # Build automation suggestions

351 suggestions = [p.suggested_automation for p in patterns if p.occurrence_count >= 2]

352

353 return ManualPatternAnalysis(

354 patterns=patterns,

355 total_manual_interventions=total_interventions,

356 automatable_count=automatable,

357 automation_suggestions=suggestions[:10],

358 )

359

360

361# Mapping from manual pattern types to configuration solutions

362 _PATTERN_TO_CONFIG: dict[str, dict[str, Any]] = {

363 "test": {

364 "hook_event": "PostToolUse",

365 "description": "Automatic test execution after code changes",

366 "suggested_config": """hooks/hooks.json:

367 "PostToolUse": [{

368 "matcher": "Edit|Write",

369 "hooks": [{

370 "type": "command",

371 "command": "pytest tests/ -x -q",

372 "timeout": 30000

373 }]

374 }]""",

375 },

376 "lint": {

377 "hook_event": "PreToolUse",

378 "description": "Automatic formatting before file writes",

379 "suggested_config": """hooks/hooks.json:

380 "PreToolUse": [{

381 "matcher": "Write|Edit",

382 "hooks": [{

383 "type": "command",

384 "command": "ruff format --check .",

385 "timeout": 10000

386 }]

387 }]""",

388 },

389 "type_check": {

390 "hook_event": "PostToolUse",

391 "description": "Type checking after code modifications",

392 "suggested_config": """hooks/hooks.json:

393 "PostToolUse": [{

394 "matcher": "Edit|Write",

395 "hooks": [{

396 "type": "command",

397 "command": "mypy --fast .",

398 "timeout": 30000

399 }]

400 }]""",

401 },

402 "build": {

403 "hook_event": "PostToolUse",

404 "description": "Build verification after changes",

405 "suggested_config": """hooks/hooks.json:

406 "PostToolUse": [{

407 "matcher": "Edit|Write",

408 "hooks": [{

409 "type": "command",

410 "command": "npm run build",

411 "timeout": 60000

412 }]

413 }]""",

414 },

415}

416

417

418 def detect_config_gaps(

419 manual_pattern_analysis: ManualPatternAnalysis,

420 project_root: Path | None = None,

421) -> ConfigGapsAnalysis:

422 """Detect configuration gaps based on manual pattern analysis.

423

424 Args:

425 manual_pattern_analysis: Results from detect_manual_patterns()

426 project_root: Project root directory (defaults to cwd)

427

428 Returns:

429 ConfigGapsAnalysis with identified gaps and coverage metrics

430 """

431 if project_root is None:

432 project_root = Path.cwd()

433

434 # Discover current configuration

435 current_hooks: list[str] = []

436 current_skills: list[str] = []

437 current_agents: list[str] = []

438

439 # Load hooks configuration

440 hooks_file = project_root / "hooks" / "hooks.json"

441 if hooks_file.exists():

442 try:

443 with open(hooks_file, encoding="utf-8") as f:

444 hooks_data = json.load(f)

445 current_hooks = list(hooks_data.get("hooks", {}).keys())

446 except Exception:

447 pass

448

449 # Scan for agents

450 agents_dir = project_root / "agents"

451 if agents_dir.is_dir():

452 for agent_file in agents_dir.glob("*.md"):

453 current_agents.append(agent_file.stem)

454

455 # Scan for skills

456 skills_dir = project_root / "skills"

457 if skills_dir.is_dir():

458 for skill_dir in skills_dir.iterdir():

459 if skill_dir.is_dir() and (skill_dir / "SKILL.md").exists():

460 current_skills.append(skill_dir.name)

461

462 # Identify gaps from manual patterns

463 gaps: list[ConfigGap] = []

464 covered_patterns = 0

465 recognized_patterns = 0

466

467 for pattern in manual_pattern_analysis.patterns:

468 config_mapping = _PATTERN_TO_CONFIG.get(pattern.pattern_type)

469 if not config_mapping:

470 continue

471

472 recognized_patterns += 1

473 hook_event = config_mapping["hook_event"]

474

475 # Check if hook event is already configured

476 if hook_event in current_hooks:

477 covered_patterns += 1

478 continue

479

480 # Determine priority based on occurrence count

481 if pattern.occurrence_count >= 10:

482 priority = "high"

483 elif pattern.occurrence_count >= 5:

484 priority = "medium"

485 else:

486 priority = "low"

487

488 gap = ConfigGap(

489 gap_type="hook",

490 description=config_mapping["description"],

491 evidence=pattern.affected_issues,

492 suggested_config=config_mapping["suggested_config"],

493 priority=priority,

494 pattern_type=pattern.pattern_type,

495 )

496 gaps.append(gap)

497

498 # Calculate coverage score based on recognized patterns only

499 coverage_score = covered_patterns / recognized_patterns if recognized_patterns > 0 else 1.0

500

501 # Sort gaps by priority (high first)

502 priority_order = {"high": 0, "medium": 1, "low": 2}

503 gaps.sort(key=lambda g: priority_order.get(g.priority, 3))

504

505 return ConfigGapsAnalysis(

506 gaps=gaps,

507 current_hooks=current_hooks,

508 current_skills=current_skills,

509 current_agents=current_agents,

510 coverage_score=coverage_score,

511 )