<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>looselips benchmark</title>
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }

    body {
      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
      background: #0f172a;
      color: #e2e8f0;
      padding: 2rem;
      max-width: 1400px;
      margin: 0 auto;
      line-height: 1.6;
    }

    h1 { color: #f8fafc; margin-bottom: 0.3rem; font-size: 1.8rem; }
    .sub { color: #94a3b8; margin-bottom: 1.5rem; font-size: 0.9rem; }

    /* -- Scorecards -- */
    .scorecards { display: flex; gap: 1rem; margin-bottom: 1.5rem; flex-wrap: wrap; }

    .scorecard {
      background: #1e293b;
      border-radius: 8px;
      padding: 0.8rem 1.2rem;
      min-width: 120px;
    }

    .scorecard .model-name {
      font-size: 0.8rem;
      color: #94a3b8;
      margin-bottom: 0.2rem;
      word-break: break-all;
    }

    .scorecard .pct { font-size: 1.8rem; font-weight: 700; }
    .scorecard .detail { font-size: 0.75rem; color: #94a3b8; }

    /* -- Crosstab -- */
    table.crosstab { border-collapse: collapse; margin-bottom: 1.5rem; font-size: 0.8rem; }

    table.crosstab th,
    table.crosstab td {
      padding: 0.3rem 0.7rem;
      text-align: right;
      border-bottom: 1px solid #334155;
    }

    table.crosstab th {
      background: #1e293b;
      color: #94a3b8;
      font-size: 0.7rem;
      text-transform: uppercase;
      letter-spacing: 0.05em;
    }

    table.crosstab td:first-child { text-align: left; color: #cbd5e1; }
    table.crosstab .ct-count { min-width: 2rem; padding: 0.3rem 0.3rem; }
    table.crosstab .ct-stat { min-width: 4.2rem; text-align: center; }

    .accuracy-good { color: #22c55e; }
    .accuracy-mid  { color: #eab308; }
    .accuracy-bad  { color: #ef4444; }

    /* -- Results table -- */
    table.results {
      border-collapse: collapse;
      margin-bottom: 1rem;
    }

    table.results th {
      background: #1e293b;
      color: #94a3b8;
      font-size: 0.7rem;
      text-transform: uppercase;
      letter-spacing: 0.05em;
      padding: 0.4rem;
      text-align: center;
      position: sticky;
      top: 0;
      z-index: 1;
      overflow: hidden;
      text-overflow: ellipsis;
    }

    table.results th.col-tc { text-align: left; }
    table.results th.col-matcher { text-align: left; }
    table.results th.col-expect { text-align: left; }

    table.results td { padding: 0; }

    .col-tc      { min-width: 14rem; }
    .col-matcher { min-width: 8rem; }
    .col-expect  { min-width: 5rem; }
    .col-model   { min-width: 4.5rem; }

    /* Testcase header row */
    tr.tc-header { cursor: pointer; }
    tr.tc-header:hover td { background: #1e293b; }

    td.tc-title {
      font-weight: 500;
      color: #f1f5f9;
      font-size: 0.9rem;
      padding: 0.6rem 0.4rem 0.15rem 0.4rem;
    }

    /* Matcher sub-rows */
    td.matcher-name {
      color: #94a3b8;
      font-size: 0.85rem;
      padding: 0.5rem 0.4rem;
    }

    td.expect-col {
      color: #64748b;
      font-size: 0.8rem;
      padding: 0.5rem 0.4rem;
    }

    /* PASS/FAIL cells -- the td itself is the colored block */
    .matcher-row td {
      height: 4rem;
    }

    td.cell-pass, td.cell-fail {
      text-align: center;
      font-weight: 600;
      font-size: 0.7rem;
      line-height: 1.3;
      vertical-align: middle;
      padding: 0.4rem 0;
    }

    td.cell-pass { background: #14532d; color: #dcfce7; }
    td.cell-fail { background: #7f1d1d; color: #fecaca; }

    td.cell-empty { background: #858585; padding: 0.4rem 0; }

    .cell-elapsed { font-weight: 400; font-size: 0.55rem; opacity: 0.5; display: block; }

    .expand-arrow {
      display: inline-block;
      transition: transform 0.15s;
      margin-right: 0.3rem;
      color: #475569;
      font-size: 0.7rem;
    }

    .expand-arrow.open { transform: rotate(90deg); }

    /* -- Detail panel -- */
    .detail-row { display: none; }
    .detail-row.open { display: table-row; }

    td.detail-cell {
      padding: 0;
      border-bottom: 2px solid #334155;
    }

    .detail-content {
      background: #1e293b;
      padding: 1rem;
      max-width: 50rem;
    }

    .detail-section { margin-bottom: 1rem; }
    .detail-section:last-child { margin-bottom: 0; }

    .detail-section h4 {
      color: #94a3b8;
      font-size: 0.75rem;
      text-transform: uppercase;
      letter-spacing: 0.05em;
      margin-bottom: 0.4rem;
    }

    .msg {
      font-family: 'SF Mono', 'Fira Code', monospace;
      font-size: 0.8rem;
      padding: 0.3rem 0;
      line-height: 1.5;
    }

    .msg-role { font-weight: 600; text-transform: uppercase; }
    .msg-role-user { color: #60a5fa; }
    .msg-role-assistant { color: #a78bfa; }

    .msg-text {
      color: #cbd5e1;
      white-space: pre-wrap;
      word-break: break-word;
      margin-left: 1rem;
      margin-bottom: 0.5rem;
    }

    .reasoning-group { margin-bottom: 0.6rem; }

    .reasoning-group-title {
      font-weight: 600;
      font-size: 0.8rem;
      color: #94a3b8;
      margin-bottom: 0.3rem;
    }

    .reasoning-box {
      background: #0f172a;
      border-radius: 4px;
      padding: 0.6rem 0.8rem;
      margin-bottom: 0.4rem;
    }

    .reasoning-label { font-weight: 600; font-size: 0.8rem; margin-bottom: 0.2rem; }
    .reasoning-label-pass { color: #22c55e; }
    .reasoning-label-fail { color: #ef4444; font-weight: 700; }
    .reasoning-text {
      font-size: 0.8rem;
      color: #cbd5e1;
      font-style: italic;
      word-break: break-word;
    }
    .reasoning-text-empty {
      font-size: 0.8rem;
      color: #475569;
      font-style: italic;
    }
    .reasoning-meta { font-size: 0.7rem; color: #64748b; margin-top: 0.2rem; }

    /* -- Suspect labels -- */
    .suspect-box {
      background: #1c1917;
      border: 1px solid #854d0e;
      border-radius: 8px;
      padding: 1rem 1.2rem;
      margin-bottom: 1.5rem;
      font-size: 0.85rem;
    }

    .suspect-box h3 {
      color: #fbbf24;
      font-size: 0.9rem;
      margin-bottom: 0.3rem;
    }

    .suspect-box p { color: #a8a29e; margin-bottom: 0.5rem; }
    .suspect-box ul { padding-left: 1.2rem; color: #d6d3d1; }
    .suspect-box li { margin-bottom: 0.3rem; }

    /* -- Footer -- */
    .footer {
      text-align: center;
      color: #475569;
      font-size: 0.75rem;
      margin-top: 2rem;
      padding-top: 1rem;
      border-top: 1px solid #1e293b;
    }
  </style>
</head>
<body>

<h1>looselips benchmark</h1>
<p class="sub">Generated {{ now }}</p>

<div class="scorecards">
{% for model in report.models %}
  {% set correct, total = report.model_score(model) %}
  {% set pct = (correct / total * 100) | int if total else 0 %}
  <div class="scorecard">
    <div class="model-name">{{ model | short_model }}</div>
    <div class="pct" style="color: {{ '#22c55e' if pct >= 80 else '#eab308' if pct >= 60 else '#ef4444' }}">{{ pct }}%</div>
    <div class="detail">{{ correct }}/{{ total }} correct</div>
  </div>
{% endfor %}
</div>

<button onclick="document.querySelectorAll('.ct-matcher').forEach(r => r.hidden = !r.hidden)" style="margin-bottom:0.5rem;cursor:pointer">Show/hide per-matcher breakdown</button>
<table class="crosstab">
  <thead>
    <tr>
      <th>Model</th><th class="ct-count">TP</th><th class="ct-count">TN</th><th class="ct-count">FP</th><th class="ct-count">FN</th><th class="ct-count">Tot</th><th class="ct-stat">Accuracy</th><th class="ct-stat">Recall</th><th class="ct-stat">Precision</th><th class="ct-stat">F1</th><th class="ct-stat">Mean</th>
    </tr>
  </thead>
  <tbody>
  {% for model in report.models %}
    {% set ct = report.crosstab(model) %}
    {% set ct_total = ct.TP + ct.TN + ct.FP + ct.FN %}
    {% set ct_ok = ct.TP + ct.TN %}
    {% set ct_pct = (ct_ok / ct_total * 100) | int if ct_total else 0 %}
    {% set ct_pos = ct.TP + ct.FN %}
    {% set ct_recall = (ct.TP / ct_pos * 100) | int if ct_pos else 0 %}
    {% set ct_predicted_pos = ct.TP + ct.FP %}
    {% set ct_precision = (ct.TP / ct_predicted_pos * 100) | int if ct_predicted_pos else 0 %}
    <tr>
      <td>{{ model | short_model }}</td>
      <td class="ct-count">{{ ct.TP }}</td><td class="ct-count">{{ ct.TN }}</td><td class="ct-count">{{ ct.FP }}</td><td class="ct-count">{{ ct.FN }}</td>
      {% set ct_expected_total = report.rows | length * report.matchers | length %}
      <td class="ct-count"{% if ct_total != ct_expected_total %} style="color:red;font-weight:bold"{% endif %}>{{ ct_total }}</td>
      <td class="ct-stat {{ 'accuracy-good' if ct_pct >= 80 else 'accuracy-mid' if ct_pct >= 60 else 'accuracy-bad' }}"><strong>{{ ct_pct }}%</strong></td>
      <td class="ct-stat {{ 'accuracy-good' if ct_recall >= 80 else 'accuracy-mid' if ct_recall >= 60 else 'accuracy-bad' }}"><strong>{{ ct_recall }}%</strong></td>
      <td class="ct-stat">{{ ct_precision }}%</td>
      {% set ct_f1 = (2 * ct.TP / (2 * ct.TP + ct.FP + ct.FN) * 100) | int if (2 * ct.TP + ct.FP + ct.FN) else 0 %}
      <td class="ct-stat">{{ ct_f1 }}%</td>
      <td class="ct-stat">{{ '%.1f' | format(report.model_mean_elapsed(model)) }}s</td>
    </tr>
    {% for matcher in report.matchers %}
    {% set mc = report.crosstab_by_matcher(model, matcher) %}
    {% set mc_total = mc.TP + mc.TN + mc.FP + mc.FN %}
    {% set mc_ok = mc.TP + mc.TN %}
    {% set mc_pct = (mc_ok / mc_total * 100) | int if mc_total else 0 %}
    {% set mc_pos = mc.TP + mc.FN %}
    {% set mc_recall = (mc.TP / mc_pos * 100) | int if mc_pos else 0 %}
    {% set mc_pp = mc.TP + mc.FP %}
    {% set mc_precision = (mc.TP / mc_pp * 100) | int if mc_pp else 0 %}
    {% set mc_f1 = (2 * mc.TP / (2 * mc.TP + mc.FP + mc.FN) * 100) | int if (2 * mc.TP + mc.FP + mc.FN) else 0 %}
    <tr class="ct-matcher" hidden style="color:#888;font-size:0.85em">
      <td style="padding-left:1.5rem">{{ matcher }}</td>
      <td class="ct-count">{{ mc.TP }}</td><td class="ct-count">{{ mc.TN }}</td><td class="ct-count">{{ mc.FP }}</td><td class="ct-count">{{ mc.FN }}</td>
      <td class="ct-count">{{ mc_total }}</td>
      <td class="ct-stat">{{ mc_pct }}%</td>
      <td class="ct-stat">{{ mc_recall }}%</td>
      <td class="ct-stat">{{ mc_precision }}%</td>
      <td class="ct-stat">{{ mc_f1 }}%</td>
      <td class="ct-stat"></td>
    </tr>
    {% endfor %}
  {% endfor %}
  </tbody>
</table>

{% set suspects = report.suspect_labels() %}
{% if suspects %}
<div class="suspect-box">
  <h3>Suspect labels</h3>
  <p>These testcase+matcher combos may have incorrect expected values.</p>
  <ul>
  {% for s in suspects %}
    <li><strong>{{ s.title }}</strong> / {{ s.matcher }} (expected {{ "match" if s.expected else "no match" }}) -- {{ s.dissenters }} disagree</li>
  {% endfor %}
  </ul>
</div>
{% endif %}

<table class="results">
  <thead>
    <tr>
      <th class="col-tc"></th>
      <th class="col-matcher">Matcher</th>
      <th class="col-expect">Expect</th>
    {% for model in report.models | reverse %}
      <th class="col-model">{{ model | short_model }}</th>
    {% endfor %}
    </tr>
  </thead>
  <tbody>
  {% for row in report.rows %}
    <tr class="tc-header" onclick="toggleDetail({{ loop.index0 }})">
      <td class="tc-title" colspan="{{ 3 + report.models | length }}">
        <span class="expand-arrow" id="arrow-{{ loop.index0 }}">&#9654;</span>
        {{ row.title }}
      </td>
    </tr>
    {% for matcher, expected in row.expectations.items() %}
    <tr class="matcher-row">
      <td></td>
      <td class="matcher-name">{{ matcher }}</td>
      <td class="expect-col">{{ "match" if expected else "no match" }}</td>
    {% for model in report.models | reverse %}
      {% set cell = row.cells.get(model, {}).get(matcher) %}
      {% if cell is not none %}
      {% set ok = (cell.found == expected) %}
      <td class="{{ 'cell-pass' if ok else 'cell-fail' }}"{% if cell.reasoning %} title="{{ cell.reasoning }}"{% endif %}>{{ "PASS" if ok else "FAIL" }}<span class="cell-elapsed">{{ '%.1f' | format(cell.elapsed) }}s</span></td>
      {% else %}
      <td class="cell-empty"></td>
      {% endif %}
    {% endfor %}
    </tr>
    {% endfor %}
    <tr class="detail-row" id="detail-{{ loop.index0 }}">
      <td class="detail-cell" colspan="{{ 3 + report.models | length }}">
        <div class="detail-content">
          <div class="detail-section">
            <h4>Conversation ({{ row.messages | length }} messages)</h4>
          {% for m in row.messages %}
            <div class="msg">
              <span class="msg-role msg-role-{{ m.role }}">{{ m.role }}:</span>
              <div class="msg-text">{{ m.text }}</div>
            </div>
          {% endfor %}
          </div>
          <div class="detail-section">
            <h4>LLM Reasoning</h4>
          {% for matcher in row.expectations %}
            <div class="reasoning-group">
              <div class="reasoning-group-title">{{ matcher }}</div>
            {% for model in report.models | reverse %}
              {% set cell = row.cells.get(model, {}).get(matcher) %}
              {% if cell is not none %}
              {% set ok = (cell.found == row.expectations[matcher]) %}
              <div class="reasoning-box">
                <div class="reasoning-label {{ 'reasoning-label-pass' if ok else 'reasoning-label-fail' }}">
                  {{ model | short_model }} -- {{ "PASS" if ok else "FAIL" }} (expected={{ row.expectations[matcher] }}, got={{ cell.found }})
                </div>
                {% if cell.reasoning %}
                <div class="reasoning-text">{{ cell.reasoning }}</div>
                {% else %}
                <div class="reasoning-text-empty">(no reasoning returned)</div>
                {% endif %}
                <div class="reasoning-meta">{{ '%.1f' | format(cell.elapsed) }}s</div>
              </div>
              {% endif %}
            {% endfor %}
            </div>
          {% endfor %}
          </div>
        </div>
      </td>
    </tr>
  {% endfor %}
  </tbody>
</table>

<div class="footer">looselips benchmark</div>

<script>
function toggleDetail(idx) {
  var row = document.getElementById('detail-' + idx);
  var arrow = document.getElementById('arrow-' + idx);
  row.classList.toggle('open');
  arrow.classList.toggle('open');
}
</script>

</body>
</html>
