name: rl-coding-agent
description: |
  Policy+RLHF composite loop for agentic coding. An outer policy loop adapts the
  coding strategy (which files, which approach) while an inner RLHF loop polishes
  each artifact to a quality threshold before the outer loop observes results.
  Reward = weighted composite of test pass rate, lint score, and live LLM-as-judge score.
  Upgrade path: when FEAT-659 (Hierarchical FSM Loops) lands, replace the refine
  state's shell subprocess with a native `loop: rl-rlhf` sub-loop state.
initial: act
context:
  reward_target: 0.85      # Convergence threshold for composite reward (0.0–1.0)
  quality_threshold: 7     # Min quality score for rl-rlhf inner loop (0–10)
  target_files: ""         # Override: space-separated file globs to act on
  test_cmd: "python -m pytest"  # Fallback if not set in ll-config.json project.test_cmd
  lint_cmd: "ruff check"        # Fallback if not set in ll-config.json project.lint_cmd
states:

  # POLICY OUTER: Execute the current coding strategy on target files.
  # Replace the echo stubs with your coding action: ll-manage-issue, a Claude
  # prompt via claude -p, a patch application, etc.
  act:
    action: |
      echo "Executing coding strategy on: ${context.target_files:-<all changed files>}"
      # Replace: run your coding action (ll-manage-issue, claude prompt, patch, etc.)
      echo "strategy_applied"
    action_type: shell
    capture: action_result
    next: refine

  # RLHF INNER: Polish the artifact until its quality meets quality_threshold.
  # Delegates to rl-rlhf as a subprocess so it gets its own convergence loop.
  # When FEAT-659 lands, replace this state with:
  #   loop: rl-rlhf
  #   context_passthrough: true
  #   on_success: observe
  #   on_failure: failed
  refine:
    action: |
      ll-loop run rl-rlhf \
        --context quality_dimension="correctness and test coverage" \
        --max-iterations 10
    action_type: shell
    capture: refine_result
    next: observe

  # POLICY OUTER: Compute composite reward from test pass rate + lint score + LLM judge score.
  # Composite formula: TEST_SCORE * 0.5 + LINT_SCORE * 0.3 + LLM_SCORE * 0.2
  observe:
    action: |
      # Load test_cmd: ll-config.json > context override > default
      TEST_CMD=$(python3 -c "
      import json, pathlib
      p = pathlib.Path('.ll/ll-config.json')
      cfg = json.loads(p.read_text()) if p.exists() else {}
      print(cfg.get('project', {}).get('test_cmd', '${context.test_cmd}'))
      ")

      # Load lint_cmd: ll-config.json > context override > default
      LINT_CMD=$(python3 -c "
      import json, pathlib
      p = pathlib.Path('.ll/ll-config.json')
      cfg = json.loads(p.read_text()) if p.exists() else {}
      print(cfg.get('project', {}).get('lint_cmd', '${context.lint_cmd}'))
      ")

      # Test score (0.0–1.0): fraction of tests that pass
      TEST_OUT=$(eval "$TEST_CMD" --tb=no -q 2>&1 | tail -1)
      PASSED=$(echo "$TEST_OUT" | grep -oE '[0-9]+ passed' | grep -oE '[0-9]+' || echo 0)
      TOTAL=$(echo "$TEST_OUT" | grep -oE '[0-9]+ (passed|failed)' | grep -oE '[0-9]+' | paste -sd+ | bc || echo 1)
      TEST_SCORE=$(python3 -c "print(round($PASSED / max($TOTAL, 1), 3))")

      # Lint score (0.0–1.0): 1 minus clamped error density
      LINT_ERRORS=$(eval "$LINT_CMD" --select E,W 2>/dev/null | grep -c '^' || echo 0)
      LINT_SCORE=$(python3 -c "print(round(max(0, 1 - $LINT_ERRORS / 10), 3))")

      # LLM quality score (0.0–1.0): live judge via Claude
      TARGETS="${context.target_files}"
      [ -z "$TARGETS" ] && TARGETS=$(git diff --name-only HEAD 2>/dev/null | tr '\n' ' ' || echo ".")
      LLM_SCORE=$(claude -p "Review $TARGETS for code quality. Rate overall quality (correctness, clarity, test coverage) as a decimal 0.0–1.0. Output ONLY the number." \
        --output-format text --no-session-persistence --dangerously-skip-permissions 2>/dev/null | \
        grep -oE '(0?\.[0-9]+|1\.0)' | head -1 || echo "0.5")
      LLM_SCORE=$(python3 -c "print(round(max(0.0, min(1.0, float('$LLM_SCORE' or '0.5'))), 3))")

      # Composite reward: tests 50% + lint 30% + LLM judge 20%
      REWARD=$(python3 -c "print(round($TEST_SCORE * 0.5 + $LINT_SCORE * 0.3 + $LLM_SCORE * 0.2, 3))")
      echo "test=$TEST_SCORE lint=$LINT_SCORE composite=$REWARD"
      echo "$REWARD"
    action_type: shell
    capture: observation
    next: score

  # POLICY OUTER: Extract numeric reward and route via convergence evaluator.
  # Routes: target → done, progress → improve, stall → act, error → failed.
  score:
    action: |
      echo "${captured.observation.output}" | tail -1 | tr -d '[:space:]'
    action_type: shell
    evaluate:
      type: convergence
      target: "${context.reward_target}"
      direction: maximize
      tolerance: 0.05
      previous: "${captured.prev_reward.output}"
    route:
      target: done
      progress: improve
      stall: act
      error: failed

  # POLICY OUTER: Update the coding strategy based on observed reward and refine output.
  # Replace the echo stubs with real strategy adaptation logic: update a prompt file,
  # adjust target_files, change the approach (e.g., switch from patching to rewriting), etc.
  improve:
    action: |
      echo "Reward achieved: ${captured.observation.output}"
      echo "Refine output: ${captured.refine_result.output}"
      # Replace: update strategy file, adjust prompt, change target files, etc.
      echo "Strategy updated based on observations."
    action_type: shell
    next: persist_reward

  # POLICY OUTER: Persist current reward for stall detection on the next iteration.
  # Emits only the numeric reward so the convergence evaluator can parse it cleanly.
  persist_reward:
    action: |
      echo "${captured.observation.output}" | tail -1 | tr -d '[:space:]'
    action_type: shell
    capture: prev_reward
    next: act

  done:
    terminal: true
  failed:
    terminal: true

max_iterations: 50
on_handoff: spawn
