{
  "paul_graham": {
    "naive": {
      "summary": {
        "n": 48,
        "mrr": 0.5190972222222222,
        "mrr_ci": [
          0.3888888888888889,
          0.6458333333333334
        ],
        "hit_rate_5": 0.6041666666666666,
        "hit_rate_5_ci": [
          0.4791666666666667,
          0.7291666666666666
        ],
        "hit_rate_1": 0.4583333333333333,
        "precision_5": 0.30833333333333335,
        "latency_p50": 16.432875,
        "avg_tokens": 629.7916666666666,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.7222222222222221,
            "hit_rate_5": 0.8,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.45,
            "hit_rate_5": 0.8,
            "n": 5
          },
          "factual": {
            "mrr": 0.9102564102564101,
            "hit_rate_5": 1.0,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 12.982917,
          "tokens": 611
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/organic-startup-ideas.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.776167,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 15.402,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/billionaires-build.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 15.747209,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.10575,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 16.407541,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 16.623208,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/mind-the-gap.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 18.543917,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/the-venture-capital-squeeze.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.270875,
          "tokens": 638
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/its-charisma-stupid.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 17.078792,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/the-submarine.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 14.446125,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/how-to-think-for-yourself.md",
            "tests/fixtures/paul_graham/orthodox-privilege.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 15.914833,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/what-youll-wish-youd-known.md",
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/what-youll-wish-youd-known.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 14.87575,
          "tokens": 638
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 14.447333,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 16.729916,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/the-hackers-guide-to-investors.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 15.729625,
          "tokens": 555
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 15.02925,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 17.228166,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/a-new-venture-animal.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/a-new-venture-animal.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 16.56075,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md",
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/how-people-get-rich-now.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.098042,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 15.453167,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md",
            "tests/fixtures/paul_graham/the-founder-visa.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 27.102125,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 16.319041,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 15.80975,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/a-new-venture-animal.md",
            "tests/fixtures/paul_graham/what-ive-learned-from-users.md",
            "tests/fixtures/paul_graham/be-good.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 17.216542,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 15.652125,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/the-need-to-read.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.927833,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/being-popular.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 16.938208,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 16.748791,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/lisp-for-web-based-applications.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 16.432875,
          "tokens": 612
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 16.998541,
          "tokens": 635
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/economic-inequality.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.3065735963827292,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 17.82725,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/why-to-start-a-startup-in-a-bad-economy.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md",
            "tests/fixtures/paul_graham/how-to-fund-a-startup.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.787584,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.637916,
          "tokens": 619
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/two-kinds-of-judgement.md",
            "tests/fixtures/paul_graham/6631372.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.585,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.195792,
          "tokens": 575
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.371375,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/being-popular.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.769584,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.626208,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.187167,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.478375,
          "tokens": 640
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.028375,
          "tokens": 611
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/the-roots-of-lisp.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.92475,
          "tokens": 587
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-list-of-n-things.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/how-not-to-die.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.166166,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/early-work.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/early-work.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.916708,
          "tokens": 623
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/is-it-worth-being-wise.md",
            "tests/fixtures/paul_graham/is-it-worth-being-wise.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.743083,
          "tokens": 639
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-do-philosophy.md",
            "tests/fixtures/paul_graham/being-a-noob.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md",
            "tests/fixtures/paul_graham/be-good.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.215458,
          "tokens": 625
        },
        {
          "competitor": "naive",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.361125,
          "tokens": 527
        }
      ]
    },
    "naive-rerank": {
      "summary": {
        "n": 48,
        "mrr": 0.5770833333333333,
        "mrr_ci": [
          0.4468489583333333,
          0.703125
        ],
        "hit_rate_5": 0.6666666666666666,
        "hit_rate_5_ci": [
          0.5208333333333334,
          0.7916666666666666
        ],
        "hit_rate_1": 0.5208333333333334,
        "precision_5": 0.3541666666666667,
        "latency_p50": 273.082917,
        "avg_tokens": 628.8333333333334,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.7966666666666666,
            "hit_rate_5": 0.9333333333333333,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.75,
            "hit_rate_5": 1.0,
            "n": 5
          },
          "factual": {
            "mrr": 0.9230769230769231,
            "hit_rate_5": 1.0,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 283.514375,
          "tokens": 612
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/organic-startup-ideas.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 289.519834,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 235.605459,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/be-good.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 237.669166,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/why-to-start-a-startup-in-a-bad-economy.md",
            "tests/fixtures/paul_graham/a-word-to-the-resourceful.md",
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.43067655807339306,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 237.17025,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 238.868333,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/organic-startup-ideas.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 273.082917,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/mind-the-gap.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 251.370042,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-convince-investors.md",
            "tests/fixtures/paul_graham/the-future-of-web-startups.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 253.819875,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 303.568416,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 322.830792,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/orthodox-privilege.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 281.304209,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/what-youll-wish-youd-known.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/beyond-smart.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 280.292917,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 335.686375,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-convince-investors.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 237.564292,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/great-hackers.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 280.454666,
          "tokens": 555
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 242.082584,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/writing-briefly.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 270.460167,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/the-pooled-risk-company-management-company.md",
            "tests/fixtures/paul_graham/be-good.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 242.731,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md",
            "tests/fixtures/paul_graham/how-people-get-rich-now.md",
            "tests/fixtures/paul_graham/dont-talk-to-corp-dev.md",
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 237.861625,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 255.045,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 299.7885,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 287.888583,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 247.518667,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/organic-startup-ideas.md",
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/be-good.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 300.449667,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 262.250583,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/write-simply.md"
          ],
          "rr": 0.2,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 266.516917,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 250.238666,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 313.766959,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/lisp-for-web-based-applications.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 320.013542,
          "tokens": 613
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/a-version-10.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 311.87725,
          "tokens": 611
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/the-refragmentation.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 271.978125,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/why-to-start-a-startup-in-a-bad-economy.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.2640681225725909,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 282.550791,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 317.165375,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 311.430666,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/startup-investing-trends.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 316.295708,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 261.835916,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 258.722708,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 260.541,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 317.92825,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 290.498542,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/are-software-patents-evil.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 281.231042,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md",
            "tests/fixtures/paul_graham/the-roots-of-lisp.md",
            "tests/fixtures/paul_graham/what-you-want-to-want.md",
            "tests/fixtures/paul_graham/five-questions-about-language-design.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 318.78025,
          "tokens": 628
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/write-simply.md",
            "tests/fixtures/paul_graham/see-randomness.md",
            "tests/fixtures/paul_graham/how-not-to-die.md",
            "tests/fixtures/paul_graham/is-there-such-a-thing-as-good-taste.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 261.629208,
          "tokens": 640
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/weird-languages.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 269.494959,
          "tokens": 577
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/is-it-worth-being-wise.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 286.050166,
          "tokens": 639
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/being-a-noob.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/how-to-do-philosophy.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/investor-herd-dynamics.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 257.4945,
          "tokens": 605
        },
        {
          "competitor": "naive-rerank",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/made-in-usa.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/how-to-be-silicon-valley.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 248.234333,
          "tokens": 415
        }
      ]
    },
    "langchain": {
      "summary": {
        "n": 48,
        "mrr": 0.515625,
        "mrr_ci": [
          0.39409722222222215,
          0.6440972222222222
        ],
        "hit_rate_5": 0.625,
        "hit_rate_5_ci": [
          0.4791666666666667,
          0.7708333333333334
        ],
        "hit_rate_1": 0.4375,
        "precision_5": 0.29583333333333334,
        "latency_p50": 20.352791,
        "avg_tokens": 464.2083333333333,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.7944444444444444,
            "hit_rate_5": 1.0,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.5,
            "hit_rate_5": 0.6,
            "n": 5
          },
          "factual": {
            "mrr": 0.7948717948717948,
            "hit_rate_5": 0.9230769230769231,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/filters-that-fight-back.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 21.8775,
          "tokens": 478
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/six-principles-for-making-new-things.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 22.604833,
          "tokens": 563
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-new-venture-animal.md",
            "tests/fixtures/paul_graham/how-to-fund-a-startup.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 22.101958,
          "tokens": 509
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 20.3185,
          "tokens": 406
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/black-swan-farming.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 19.385083,
          "tokens": 505
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/a-version-10.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 19.923334,
          "tokens": 512
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/organic-startup-ideas.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 18.609375,
          "tokens": 305
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/mind-the-gap.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/mind-the-gap.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 20.245042,
          "tokens": 479
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-make-wealth.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/how-to-fund-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 17.354166,
          "tokens": 486
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 20.680833,
          "tokens": 439
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 17.068292,
          "tokens": 463
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 21.027292,
          "tokens": 456
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/how-to-work-hard.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 18.101292,
          "tokens": 349
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 19.122208,
          "tokens": 397
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 17.837208,
          "tokens": 530
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/great-hackers.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 16.343,
          "tokens": 422
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 20.8845,
          "tokens": 377
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 19.345,
          "tokens": 440
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-new-venture-animal.md",
            "tests/fixtures/paul_graham/how-not-to-die.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 18.764792,
          "tokens": 449
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/mind-the-gap.md",
            "tests/fixtures/paul_graham/how-people-get-rich-now.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 18.898916,
          "tokens": 521
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 21.723875,
          "tokens": 527
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 22.687542,
          "tokens": 516
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/billionaires-build.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 20.615,
          "tokens": 356
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/jessica-livingston.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 21.2615,
          "tokens": 537
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/what-ive-learned-from-users.md",
            "tests/fixtures/paul_graham/how-y-combinator-started.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/billionaires-build.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 19.842208,
          "tokens": 492
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/the-word-hacker.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/hackers-and-painters.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 20.895375,
          "tokens": 444
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/the-need-to-read.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/write-simply.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.43067655807339306,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 21.359167,
          "tokens": 580
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/hackers-and-painters.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 22.652583,
          "tokens": 418
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 21.493875,
          "tokens": 500
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/lisp-for-web-based-applications.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 22.633833,
          "tokens": 582
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 22.400666,
          "tokens": 596
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/economic-inequality.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 22.949,
          "tokens": 430
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/how-to-fund-a-startup.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 21.55075,
          "tokens": 521
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 21.014125,
          "tokens": 475
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/two-kinds-of-judgement.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 21.647875,
          "tokens": 559
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/the-ronco-principle.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.626791,
          "tokens": 308
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/are-software-patents-evil.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.271375,
          "tokens": 327
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/the-venture-capital-squeeze.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.430125,
          "tokens": 459
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 20.819625,
          "tokens": 511
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 20.352791,
          "tokens": 431
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/five-questions-about-language-design.md",
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/subject-airbnb.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.231083,
          "tokens": 604
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.469667,
          "tokens": 433
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.644416,
          "tokens": 246
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/write-simply.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.905292,
          "tokens": 363
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/crazy-new-ideas.md",
            "tests/fixtures/paul_graham/early-work.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 21.784916,
          "tokens": 372
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/what-youll-wish-youd-known.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.544292,
          "tokens": 601
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-do-philosophy.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md",
            "tests/fixtures/paul_graham/the-ronco-principle.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 20.712834,
          "tokens": 558
        },
        {
          "competitor": "langchain",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/hackers-and-painters.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.318125,
          "tokens": 450
        }
      ]
    },
    "candlekeep-simple": {
      "summary": {
        "n": 48,
        "mrr": 0.37916666666666665,
        "mrr_ci": [
          0.23750000000000002,
          0.5208333333333334
        ],
        "hit_rate_5": 0.6875,
        "hit_rate_5_ci": [
          0.5416666666666666,
          0.8125
        ],
        "hit_rate_1": 0.6666666666666666,
        "precision_5": 0.5385416666666667,
        "latency_p50": 111.0435,
        "avg_tokens": 652.2291666666666,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.9333333333333333,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.41333333333333333,
            "hit_rate_5": 0.4666666666666667,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.8,
            "hit_rate_5": 0.8,
            "n": 5
          },
          "factual": {
            "mrr": 0.6153846153846154,
            "hit_rate_5": 0.6153846153846154,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 107.57675,
          "tokens": 407
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 102.704,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 303.75925,
          "tokens": 1501
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 114.468833,
          "tokens": 547
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 397.651666,
          "tokens": 2073
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 79.24925,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 88.363417,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 103.315166,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 118.874583,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 91.783791,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 99.402583,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 111.0435,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 105.573,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 713.710333,
          "tokens": 1925
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 138.554792,
          "tokens": 138
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 100.378458,
          "tokens": 270
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 73.25175,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 88.154208,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md"
          ],
          "rr": 0.2,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 546.101958,
          "tokens": 2913
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 115.35325,
          "tokens": 689
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 177.897375,
          "tokens": 2652
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/the-founder-visa.md",
            "tests/fixtures/paul_graham/why-to-start-a-startup-in-a-bad-economy.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 708.154917,
          "tokens": 1660
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 119.403334,
          "tokens": 1220
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 98.0575,
          "tokens": 417
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/five-founders.md",
            "tests/fixtures/paul_graham/what-ive-learned-from-users.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 774.101291,
          "tokens": 1227
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 100.270958,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-simply.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 61.368625,
          "tokens": 843
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 171.775584,
          "tokens": 3025
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 378.033459,
          "tokens": 2058
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 738.757417,
          "tokens": 1227
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-like-you-talk.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 62.5185,
          "tokens": 1203
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 241.005916,
          "tokens": 4052
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 75.3215,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 109.861375,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 105.136958,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 108.239084,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 346.365875,
          "tokens": 1260
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 101.786875,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 115.657709,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 70.02025,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 104.826208,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 60.234667,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 99.339875,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 124.832834,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 116.666584,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 141.188875,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 130.296917,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 138.909292,
          "tokens": 0
        }
      ]
    },
    "candlekeep-hybrid": {
      "summary": {
        "n": 48,
        "mrr": 0.6423611111111112,
        "mrr_ci": [
          0.5,
          0.7708333333333334
        ],
        "hit_rate_5": 0.6666666666666666,
        "hit_rate_5_ci": [
          0.5208333333333334,
          0.7916666666666666
        ],
        "hit_rate_1": 0.625,
        "precision_5": 0.24722222222222223,
        "latency_p50": 661.97475,
        "avg_tokens": 2335.7708333333335,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.888888888888889,
            "hit_rate_5": 0.9333333333333333,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.9,
            "hit_rate_5": 1.0,
            "n": 5
          },
          "factual": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 726.899083,
          "tokens": 2894
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/snapshot-viaweb-june-1998.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/what-doesnt-seem-like-work.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 816.924958,
          "tokens": 1648
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/jessica-livingston.md",
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 764.398625,
          "tokens": 1234
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/five-founders.md",
            "tests/fixtures/paul_graham/be-good.md",
            "tests/fixtures/paul_graham/the-18-mistakes-that-kill-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 661.97475,
          "tokens": 1775
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/subject-airbnb.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 590.242292,
          "tokens": 2188
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-need-to-read.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 480.459375,
          "tokens": 3807
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 596.1585,
          "tokens": 4393
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/great-hackers.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/mind-the-gap.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 635.786625,
          "tokens": 2569
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/earnestness.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/economic-inequality.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 587.838,
          "tokens": 2174
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/how-to-do-what-you-love.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/keep-your-identity-small.md",
            "tests/fixtures/paul_graham/how-to-think-for-yourself.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 627.757125,
          "tokens": 2331
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/the-submarine.md",
            "tests/fixtures/paul_graham/what-the-bubble-got-right.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 708.590458,
          "tokens": 1634
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/mind-the-gap.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 594.006334,
          "tokens": 1497
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/how-to-work-hard.md",
            "tests/fixtures/paul_graham/what-youll-wish-youd-known.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 804.153584,
          "tokens": 1933
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/the-new-funding-landscape.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 793.904459,
          "tokens": 1379
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/be-good.md",
            "tests/fixtures/paul_graham/jessica-livingston.md",
            "tests/fixtures/paul_graham/news-from-the-front.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 918.423375,
          "tokens": 1226
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/hackers-and-painters.md",
            "tests/fixtures/paul_graham/the-power-of-the-marginal.md",
            "tests/fixtures/paul_graham/undergraduation.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 753.599583,
          "tokens": 1144
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/why-tv-lost.md",
            "tests/fixtures/paul_graham/how-to-make-pittsburgh-a-startup-hub.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 705.78825,
          "tokens": 1970
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/how-to-write-usefully.md",
            "tests/fixtures/paul_graham/being-popular.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 729.098958,
          "tokens": 2172
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 862.702125,
          "tokens": 3061
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md",
            "tests/fixtures/paul_graham/mind-the-gap.md",
            "tests/fixtures/paul_graham/how-people-get-rich-now.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/mind-the-gap.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 790.864625,
          "tokens": 2046
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/beyond-smart.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/a-version-10.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 541.96525,
          "tokens": 4007
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/heresy.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md",
            "tests/fixtures/paul_graham/after-credentials.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 661.802917,
          "tokens": 2614
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/why-smart-people-have-bad-ideas.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 443.229333,
          "tokens": 2455
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 449.805166,
          "tokens": 2405
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/how-y-combinator-started.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/jessica-livingston.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 931.902167,
          "tokens": 1233
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/return-of-the-mac.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 770.439958,
          "tokens": 1894
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-simply.md",
            "tests/fixtures/paul_graham/the-submarine.md",
            "tests/fixtures/paul_graham/how-to-disagree.md",
            "tests/fixtures/paul_graham/relentlessly-resourceful.md",
            "tests/fixtures/paul_graham/how-to-disagree.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 641.985709,
          "tokens": 1792
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/putting-ideas-into-words.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6666666666666666,
          "latency_ms": 308.7685,
          "tokens": 3713
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/a-local-revolution.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 736.0155,
          "tokens": 1778
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/lisp-for-web-based-applications.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 808.952791,
          "tokens": 2553
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md",
            "tests/fixtures/paul_graham/a-version-10.md",
            "tests/fixtures/paul_graham/the-age-of-the-essay.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 556.2785,
          "tokens": 5160
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/the-refragmentation.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 468.118917,
          "tokens": 8383
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 522.331292,
          "tokens": 4537
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/filters-that-fight-back.md",
            "tests/fixtures/paul_graham/the-refragmentation.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-four-quadrants-of-conformism.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 599.380125,
          "tokens": 2452
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/the-submarine.md",
            "tests/fixtures/paul_graham/two-kinds-of-judgement.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 870.30525,
          "tokens": 3305
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/succinctness-is-power.md",
            "tests/fixtures/paul_graham/let-the-other-95-of-great-programmers-in.md",
            "tests/fixtures/paul_graham/the-bus-ticket-theory-of-genius.md",
            "tests/fixtures/paul_graham/undergraduation.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 694.752417,
          "tokens": 2206
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/frighteningly-ambitious-startup-ideas.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/web-20.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 670.904583,
          "tokens": 1217
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/snapshot-viaweb-june-1998.md",
            "tests/fixtures/paul_graham/how-people-get-rich-now.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/why-twitter-is-a-big-deal.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 715.502125,
          "tokens": 1495
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/is-it-worth-being-wise.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/lies-we-tell-kids.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 818.516917,
          "tokens": 1551
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/cities-and-ambition.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/what-ive-learned-from-users.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 596.874125,
          "tokens": 2188
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/being-popular.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md",
            "tests/fixtures/paul_graham/being-popular.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 689.071083,
          "tokens": 1495
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-lesson-to-unlearn.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 628.927709,
          "tokens": 2471
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-hundred-year-language.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/the-refragmentation.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 654.723292,
          "tokens": 1502
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/how-not-to-die.md",
            "tests/fixtures/paul_graham/write-simply.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 974.756625,
          "tokens": 720
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/the-lesson-to-unlearn.md",
            "tests/fixtures/paul_graham/how-to-be-an-expert-in-a-changing-world.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 490.637666,
          "tokens": 1196
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-lesson-to-unlearn.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/what-you-cant-say.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/subject-airbnb.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 534.571834,
          "tokens": 915
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/being-a-noob.md",
            "tests/fixtures/paul_graham/how-to-do-philosophy.md",
            "tests/fixtures/paul_graham/be-good.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 658.721709,
          "tokens": 2559
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/subject-airbnb.md",
            "tests/fixtures/paul_graham/taste-for-makers.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/how-to-be-silicon-valley.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 557.498375,
          "tokens": 1246
        }
      ]
    },
    "ablation-no-bardic": {
      "summary": {
        "n": 48,
        "mrr": 0.2881944444444445,
        "mrr_ci": [
          0.17013888888888887,
          0.4236111111111111
        ],
        "hit_rate_5": 0.6041666666666666,
        "hit_rate_5_ci": [
          0.4583333333333333,
          0.7291666666666666
        ],
        "hit_rate_1": 0.5625,
        "precision_5": 0.48715277777777777,
        "latency_p50": 132.021583,
        "avg_tokens": 304.3541666666667,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.9333333333333333,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.26666666666666666,
            "hit_rate_5": 0.26666666666666666,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.26666666666666666,
            "hit_rate_5": 0.4,
            "n": 5
          },
          "factual": {
            "mrr": 0.6538461538461539,
            "hit_rate_5": 0.6923076923076923,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 248.358458,
          "tokens": 997
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 104.582084,
          "tokens": 256
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 216.824084,
          "tokens": 639
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 135.05875,
          "tokens": 256
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/schlep-blindness.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 735.855666,
          "tokens": 896
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 88.152875,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 108.568709,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 114.811417,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 133.472167,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 94.225875,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 117.382291,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 139.545791,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 123.128458,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/ideas-for-startups.md",
            "tests/fixtures/paul_graham/why-its-safe-for-founders-to-be-nice.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 764.725167,
          "tokens": 1231
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 202.135792,
          "tokens": 639
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md",
            "tests/fixtures/paul_graham/undergraduation.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 154.2675,
          "tokens": 300
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 83.080916,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 109.289,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-hardest-lessons-for-startups-to-learn.md",
            "tests/fixtures/paul_graham/startup-growth.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/a-new-venture-animal.md",
            "tests/fixtures/paul_graham/the-pooled-risk-company-management-company.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 738.257375,
          "tokens": 1024
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 134.252875,
          "tokens": 640
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 88.086875,
          "tokens": 1412
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/where-to-see-silicon-valley.md",
            "tests/fixtures/paul_graham/how-to-make-wealth.md",
            "tests/fixtures/paul_graham/after-credentials.md",
            "tests/fixtures/paul_graham/be-good.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 821.969792,
          "tokens": 767
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 131.229584,
          "tokens": 1155
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 132.021583,
          "tokens": 384
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/early-work.md",
            "tests/fixtures/paul_graham/how-to-convince-investors.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 556.834666,
          "tokens": 639
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 128.1905,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 74.050541,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 128.320625,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.3333333333333333,
          "latency_ms": 191.878833,
          "tokens": 511
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/lisp-for-web-based-applications.md",
            "tests/fixtures/paul_graham/what-i-worked-on.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/revenge-of-the-nerds.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.3065735963827292,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 829.061125,
          "tokens": 1197
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 77.137792,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 173.457916,
          "tokens": 769
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 127.852041,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 114.417375,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 127.561542,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 141.542625,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 253.511292,
          "tokens": 897
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 100.692208,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 124.860042,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 84.001083,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 124.914458,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 80.308792,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 120.253292,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 137.487958,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 133.50525,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 195.984875,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 145.553709,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 135.451458,
          "tokens": 0
        }
      ]
    },
    "ablation-no-arcane": {
      "summary": {
        "n": 48,
        "mrr": 0.40277777777777773,
        "mrr_ci": [
          0.2708333333333333,
          0.5416666666666666
        ],
        "hit_rate_5": 0.7083333333333334,
        "hit_rate_5_ci": [
          0.5833333333333334,
          0.8333333333333334
        ],
        "hit_rate_1": 0.6875,
        "precision_5": 0.5836805555555555,
        "latency_p50": 18.5885,
        "avg_tokens": 212.95833333333334,
        "by_category": {
          "adversarial": {
            "mrr": 0.0,
            "hit_rate_5": 0.9333333333333333,
            "n": 15
          },
          "conceptual": {
            "mrr": 0.4222222222222223,
            "hit_rate_5": 0.4666666666666667,
            "n": 15
          },
          "cross-essay": {
            "mrr": 0.8,
            "hit_rate_5": 0.8,
            "n": 5
          },
          "factual": {
            "mrr": 0.6923076923076923,
            "hit_rate_5": 0.6923076923076923,
            "n": 13
          }
        }
      },
      "details": [
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is the key insight that Paul Graham describes about spam filtering?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-plan-for-spam.md",
            "tests/fixtures/paul_graham/better-bayesian-filtering.md",
            "tests/fixtures/paul_graham/a-plan-for-spam.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6666666666666666,
          "latency_ms": 57.63675,
          "tokens": 407
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What was novel about the web-based application that Paul Graham and Robert Morris built at Viaweb?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.372583,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is the most common unscalable thing that founders have to do at the start of a startup?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/do-things-that-dont-scale.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 71.022583,
          "tokens": 545
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what are the three things that startups need to create to be successful?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 22.480583,
          "tokens": 136
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Why does Paul Graham think that mean people tend to fail as startup founders?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/what-startups-are-really-like.md",
            "tests/fixtures/paul_graham/mean-people-fail.md",
            "tests/fixtures/paul_graham/why-to-not-not-start-a-startup.md",
            "tests/fixtures/paul_graham/before-the-startup.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 140.602792,
          "tokens": 683
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How did the teaching of writing become mixed together with the study of literature, according to Paul Graham?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.552834,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the key to getting good startup ideas?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 17.342834,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "According to Paul Graham, what is the key difference between a low-tech and high-tech society in terms of variation in productivity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.530959,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the economic proposition of starting a startup?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.787583,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does Paul Graham say is the problem with having religion and politics as part of one's identity?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.79225,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Did we actually dress like that?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.882708,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What would someone coming back to visit us in a time machine have to be careful not to say?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.065,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is the recipe for genius?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 18.203542,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Why is raising money so brutal for startups?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md",
            "tests/fixtures/paul_graham/how-to-raise-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 177.022084,
          "tokens": 683
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is the most important predictor of startup success?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-anatomy-of-determination.md",
            "tests/fixtures/paul_graham/a-fundraising-survival-guide.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 53.114125,
          "tokens": 276
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How can a computer science undergraduate become a good hacker?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/undergraduation.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 15.9935,
          "tokens": 134
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What message does each city send to its residents?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.868042,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How much rewriting does Paul Graham do for his essays?",
          "category": "factual",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.781458,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Why don't more startups become as successful as Google?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-future-of-web-startups.md",
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/why-there-arent-more-googles.md",
            "tests/fixtures/paul_graham/a-new-venture-animal.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 159.092125,
          "tokens": 687
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How do most fortunes get lost?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-lose-time-and-money.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 16.24275,
          "tokens": 137
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is the main difference between design and research according to the essay?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md",
            "tests/fixtures/paul_graham/design-and-research.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 100.063333,
          "tokens": 679
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What are some of the advantages that startups get from being located in the US according to the essay?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md",
            "tests/fixtures/paul_graham/why-to-start-a-startup-in-a-bad-economy.md",
            "tests/fixtures/paul_graham/the-founder-visa.md",
            "tests/fixtures/paul_graham/why-to-move-to-a-startup-hub.md",
            "tests/fixtures/paul_graham/why-startups-condense-in-america.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 228.979625,
          "tokens": 692
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Why does the author recommend changing your company's name if you don't have the .com domain for it?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md",
            "tests/fixtures/paul_graham/change-your-name.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 80.736041,
          "tokens": 540
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What surprised the architect Kate the most about the startup world?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/what-kate-saw-in-silicon-valley.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 18.112084,
          "tokens": 139
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does the author say is the key thing that Y Combinator looks for in founders?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/billionaires-build.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/five-founders.md",
            "tests/fixtures/paul_graham/what-we-look-for-in-founders.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 172.582334,
          "tokens": 682
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What was the canonical hacker's computer before the rise of Intel/Microsoft boxes?",
          "category": "factual",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.294042,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to writing simply and effectively?",
          "category": "conceptual",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-simply.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 18.5885,
          "tokens": 36
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What does the author say is the key to holding a program in one's head?",
          "category": "conceptual",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md",
            "tests/fixtures/paul_graham/holding-a-program-in-ones-head.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 147.163625,
          "tokens": 694
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What are the key factors that contribute to the success of a startup according to the essay 'How to Start a Startup'?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/how-to-start-a-startup.md",
            "tests/fixtures/paul_graham/startups-in-13-sentences.md",
            "tests/fixtures/paul_graham/before-the-startup.md",
            "tests/fixtures/paul_graham/how-to-get-startup-ideas.md",
            "tests/fixtures/paul_graham/a-students-guide-to-startups.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 94.392167,
          "tokens": 685
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How does the essay 'Beating the Averages' describe the use of the Lisp programming language in the Viaweb startup, and how does this relate to the advice given in 'Great Hackers' about learning Lisp?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md",
            "tests/fixtures/paul_graham/beating-the-averages.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 173.462083,
          "tokens": 624
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What are the key differences between written and spoken language described in the essays 'The Age of the Essay' and 'Write Like You Talk', and how do these differences impact the effectiveness of writing?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md",
            "tests/fixtures/paul_graham/write-like-you-talk.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 76.169625,
          "tokens": 543
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How do the essays 'How to Make Wealth' and 'Inequality and Risk' discuss the relationship between economic inequality and productivity, and what strategies are proposed for addressing inequality?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/economic-inequality.md",
            "tests/fixtures/paul_graham/inequality-and-risk.md"
          ],
          "rr": 1.0,
          "ndcg_5": 0.6131471927654584,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 93.664292,
          "tokens": 641
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What insights do the essays 'Mean People Fail' and 'The Anatomy of Determination' provide about the relationship between personal character traits and entrepreneurial success?",
          "category": "cross-essay",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 19.940584,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "how to configure nginx reverse proxy for kubernetes",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 18.556625,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "best practices for PostgreSQL index optimization",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 17.56925,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "machine learning model training with PyTorch distributed",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 17.928708,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "OWASP top 10 web application security vulnerabilities 2024",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md",
            "tests/fixtures/paul_graham/web-20.md",
            "tests/fixtures/paul_graham/the-other-road-ahead.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 110.042209,
          "tokens": 579
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "terraform module for AWS VPC with private subnets",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 17.192458,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "how to implement OAuth 2.0 PKCE flow in React",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 17.001667,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Redis cluster sharding with consistent hashing",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 16.997334,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "GraphQL subscription implementation with WebSocket",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 15.293833,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "differential privacy in federated learning systems",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 13.5115,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "quantum computing applications in molecular biology",
          "category": "adversarial",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 15.248584,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What did Paul Graham say about cooking Italian food?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 17.181875,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Paul Graham's advice on learning to play guitar",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 18.769333,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "How does Paul Graham recommend training for marathons?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 18.578792,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "What is Paul Graham's opinion on cryptocurrency mining?",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 20.706792,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "paul_graham",
          "query": "Paul Graham's guide to home renovation and interior design",
          "category": "adversarial",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 24.176916,
          "tokens": 0
        }
      ]
    }
  },
  "diagnostic": {
    "naive": {
      "summary": {
        "n": 30,
        "mrr": 0.6888888888888888,
        "mrr_ci": [
          0.5222222222222223,
          0.8444444444444446
        ],
        "hit_rate_5": 0.7333333333333333,
        "hit_rate_5_ci": [
          0.5666666666666667,
          0.8666666666666667
        ],
        "hit_rate_1": 0.6666666666666666,
        "precision_5": 0.4066666666666667,
        "latency_p50": 13.691458,
        "avg_tokens": 264.96666666666664,
        "by_category": {
          "context-dependent": {
            "mrr": 0.7222222222222222,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.7222222222222223,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 9.737834,
          "tokens": 358
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/document-processing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 13.479041,
          "tokens": 35
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/nosql-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 12.118042,
          "tokens": 270
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/data-lake-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 13.093625,
          "tokens": 369
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 14.298333,
          "tokens": 224
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/document-processing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 13.06125,
          "tokens": 160
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 13.002167,
          "tokens": 558
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 16.093958,
          "tokens": 283
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 13.020208,
          "tokens": 281
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/scale_docs/vpn-tunneling.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.008292,
          "tokens": 365
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/linux-networking-config.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 13.479416,
          "tokens": 211
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/network-security-hardening.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.329,
          "tokens": 361
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/owasp-top-10.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.519375,
          "tokens": 342
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 17.756167,
          "tokens": 260
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/data-pipeline-design.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 18.301125,
          "tokens": 342
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.548166,
          "tokens": 290
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/vector-databases.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 14.862125,
          "tokens": 208
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 16.79875,
          "tokens": 355
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 14.263417,
          "tokens": 46
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 14.668541,
          "tokens": 194
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/testing-strategies.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 13.151708,
          "tokens": 119
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 12.9955,
          "tokens": 164
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 13.74475,
          "tokens": 505
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/cloud-storage-options.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 13.691458,
          "tokens": 450
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.7915,
          "tokens": 121
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/database-backup-recovery.md",
            "tests/fixtures/scale_docs/chaos-engineering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.570666,
          "tokens": 151
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/chaos-engineering.md",
            "tests/fixtures/scale_docs/infrastructure-as-code.md",
            "tests/fixtures/scale_docs/monitoring-observability.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/network-troubleshooting.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.03,
          "tokens": 158
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/data-serialization.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.939625,
          "tokens": 87
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/linux-process-management.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/design-patterns-structural.md",
            "tests/fixtures/scale_docs/cryptography-basics.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.469292,
          "tokens": 524
        },
        {
          "competitor": "naive",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.014417,
          "tokens": 158
        }
      ]
    },
    "naive-rerank": {
      "summary": {
        "n": 30,
        "mrr": 0.47055555555555556,
        "mrr_ci": [
          0.3194027777777778,
          0.6316944444444443
        ],
        "hit_rate_5": 0.6333333333333333,
        "hit_rate_5_ci": [
          0.4666666666666667,
          0.8
        ],
        "hit_rate_1": 0.36666666666666664,
        "precision_5": 0.30666666666666664,
        "latency_p50": 283.657833,
        "avg_tokens": 389.8,
        "by_category": {
          "context-dependent": {
            "mrr": 0.5416666666666666,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.75,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 0.6388888888888888,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 0.4222222222222222,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/ssh-configuration.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 275.232375,
          "tokens": 631
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 431.179458,
          "tokens": 237
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 274.181833,
          "tokens": 517
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 315.481583,
          "tokens": 420
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 371.958,
          "tokens": 312
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/functional-programming.md",
            "tests/fixtures/sample_docs/document-processing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 158.377,
          "tokens": 217
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/monitoring-observability.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 312.284375,
          "tokens": 463
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 306.067416,
          "tokens": 276
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 539.052625,
          "tokens": 281
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/tcp-ip-fundamentals.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/service-mesh.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 487.196833,
          "tokens": 545
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 488.030625,
          "tokens": 488
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 172.96625,
          "tokens": 516
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 317.2985,
          "tokens": 314
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/cdn-architecture.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 221.278459,
          "tokens": 517
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/database-migrations.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 247.227916,
          "tokens": 588
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 266.352417,
          "tokens": 418
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/sample_docs/vector-databases.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 266.962791,
          "tokens": 390
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 338.119208,
          "tokens": 629
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/load-balancing.md",
            "tests/fixtures/scale_docs/cdn-architecture.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 205.399,
          "tokens": 482
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 225.30375,
          "tokens": 285
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/testing-strategies.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 312.5485,
          "tokens": 247
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/api-design.md"
          ],
          "rr": 0.2,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 290.438417,
          "tokens": 468
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 291.690833,
          "tokens": 401
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/dependency-injection.md",
            "tests/fixtures/scale_docs/container-fundamentals.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 205.069583,
          "tokens": 530
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 171.849458,
          "tokens": 202
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/database-backup-recovery.md",
            "tests/fixtures/scale_docs/network-security-hardening.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 197.637875,
          "tokens": 151
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/infrastructure-as-code.md",
            "tests/fixtures/scale_docs/linux-networking-config.md",
            "tests/fixtures/scale_docs/design-patterns-structural.md",
            "tests/fixtures/scale_docs/event-driven-architecture.md",
            "tests/fixtures/scale_docs/chaos-engineering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 283.657833,
          "tokens": 258
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/cryptography-basics.md",
            "tests/fixtures/scale_docs/functional-programming.md",
            "tests/fixtures/scale_docs/data-lake-architecture.md",
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/scale_docs/shell-scripting.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 247.36725,
          "tokens": 278
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/cryptography-basics.md",
            "tests/fixtures/scale_docs/cryptography-basics.md",
            "tests/fixtures/scale_docs/apache-kafka.md",
            "tests/fixtures/scale_docs/testing-strategies.md",
            "tests/fixtures/scale_docs/tcp-ip-fundamentals.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 367.269791,
          "tokens": 501
        },
        {
          "competitor": "naive-rerank",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/shell-scripting.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/scale_docs/stream-processing.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 209.533875,
          "tokens": 132
        }
      ]
    },
    "langchain": {
      "summary": {
        "n": 30,
        "mrr": 0.575,
        "mrr_ci": [
          0.4083333333333333,
          0.7416666666666667
        ],
        "hit_rate_5": 0.6333333333333333,
        "hit_rate_5_ci": [
          0.4666666666666667,
          0.8
        ],
        "hit_rate_1": 0.5333333333333333,
        "precision_5": 0.3066666666666667,
        "latency_p50": 14.596541,
        "avg_tokens": 355.73333333333335,
        "by_category": {
          "context-dependent": {
            "mrr": 0.5833333333333334,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.8333333333333334,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 0.9166666666666666,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 0.5416666666666666,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/ssh-configuration.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 12.979375,
          "tokens": 437
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/database-sharding.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 12.985708,
          "tokens": 417
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 13.562583,
          "tokens": 423
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/grpc-protocol.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/owasp-top-10.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 12.475666,
          "tokens": 441
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/event-driven-architecture.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 14.675667,
          "tokens": 329
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/functional-programming.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/document-processing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.596541,
          "tokens": 401
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 14.746625,
          "tokens": 521
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/sample_docs/api-design.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 15.964875,
          "tokens": 410
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/postgresql-internals.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 14.976625,
          "tokens": 340
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.620167,
          "tokens": 347
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 16.749333,
          "tokens": 421
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/network-security-hardening.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 16.390792,
          "tokens": 312
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 16.517667,
          "tokens": 432
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 17.535667,
          "tokens": 482
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 15.445666,
          "tokens": 106
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/network-security-hardening.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.926209,
          "tokens": 341
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/scale_docs/nosql-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 14.932291,
          "tokens": 479
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 17.098041,
          "tokens": 321
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/dns-resolution.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 13.302584,
          "tokens": 360
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 13.303375,
          "tokens": 372
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-lake-architecture.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.576416,
          "tokens": 398
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 12.825625,
          "tokens": 231
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 12.811875,
          "tokens": 379
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/dependency-injection.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 13.287584,
          "tokens": 474
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.205958,
          "tokens": 303
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/event-driven-architecture.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/scale_docs/well-architected-framework.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.011208,
          "tokens": 261
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/event-driven-architecture.md",
            "tests/fixtures/scale_docs/error-handling-patterns.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/cdn-architecture.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.728459,
          "tokens": 284
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/cryptography-basics.md",
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/event-driven-architecture.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.819542,
          "tokens": 141
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/design-patterns-creational.md",
            "tests/fixtures/scale_docs/linux-process-management.md",
            "tests/fixtures/scale_docs/design-patterns-creational.md",
            "tests/fixtures/scale_docs/database-sharding.md",
            "tests/fixtures/scale_docs/cryptography-basics.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 16.681583,
          "tokens": 383
        },
        {
          "competitor": "langchain",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/penetration-testing-methodology.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.624,
          "tokens": 126
        }
      ]
    },
    "candlekeep-simple": {
      "summary": {
        "n": 30,
        "mrr": 0.5983333333333333,
        "mrr_ci": [
          0.43495833333333334,
          0.765
        ],
        "hit_rate_5": 0.8666666666666667,
        "hit_rate_5_ci": [
          0.7333333333333333,
          0.9666666666666667
        ],
        "hit_rate_1": 0.7666666666666667,
        "precision_5": 0.5616666666666668,
        "latency_p50": 115.9155,
        "avg_tokens": 1306.2,
        "by_category": {
          "context-dependent": {
            "mrr": 0.7083333333333334,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.7000000000000001,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 0.9166666666666666,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 0.6666666666666666,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 188.080542,
          "tokens": 1002
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/database-sharding.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/load-balancing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 250.030333,
          "tokens": 4286
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 50.298084,
          "tokens": 706
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 487.743584,
          "tokens": 1810
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 53.06225,
          "tokens": 893
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 115.9155,
          "tokens": 712
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 273.863708,
          "tokens": 2321
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 71.56025,
          "tokens": 854
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 50.70625,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/vpn-tunneling.md",
            "tests/fixtures/scale_docs/tls-ssl.md"
          ],
          "rr": 0.2,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 707.807708,
          "tokens": 2714
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/container-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 139.484083,
          "tokens": 2823
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/network-security-hardening.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 412.136291,
          "tokens": 4888
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/owasp-top-10.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 636.387709,
          "tokens": 1396
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 43.086083,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 528.82675,
          "tokens": 1411
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/cost-optimization.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.43067655807339306,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 518.057792,
          "tokens": 2313
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 104.33375,
          "tokens": 804
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 213.08125,
          "tokens": 3793
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 150.407417,
          "tokens": 1100
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 42.44675,
          "tokens": 993
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 59.067542,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 360.083542,
          "tokens": 3127
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 146.589833,
          "tokens": 1240
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 43.606791,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 58.004458,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 61.36625,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 75.03875,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 68.230708,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 70.352917,
          "tokens": 0
        },
        {
          "competitor": "candlekeep-simple",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 49.268042,
          "tokens": 0
        }
      ]
    },
    "candlekeep-hybrid": {
      "summary": {
        "n": 30,
        "mrr": 0.6861111111111111,
        "mrr_ci": [
          0.5304861111111112,
          0.8333333333333334
        ],
        "hit_rate_5": 0.7666666666666667,
        "hit_rate_5_ci": [
          0.6,
          0.9
        ],
        "hit_rate_1": 0.6333333333333333,
        "precision_5": 0.23333333333333334,
        "latency_p50": 447.04275,
        "avg_tokens": 4170.333333333333,
        "by_category": {
          "context-dependent": {
            "mrr": 0.6388888888888888,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "negation": {
            "mrr": 0.9166666666666666,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 0.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 0.875,
            "hit_rate_5": 1.0,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 551.240625,
          "tokens": 2686
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 221.501875,
          "tokens": 2236
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/nosql-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 168.17725,
          "tokens": 2321
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 520.23575,
          "tokens": 4757
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/incident-response.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 559.629083,
          "tokens": 2383
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/scale_docs/sql-query-optimization.md",
            "tests/fixtures/sample_docs/document-processing.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 464.050083,
          "tokens": 2435
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/linux-logging.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/data-pipeline-design.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 492.134042,
          "tokens": 1678
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/helm-charts.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/data-pipeline-design.md",
            "tests/fixtures/scale_docs/redis-data-structures.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 323.356708,
          "tokens": 2668
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/postgresql-internals.md",
            "tests/fixtures/scale_docs/linux-permissions.md",
            "tests/fixtures/scale_docs/connection-pooling.md",
            "tests/fixtures/scale_docs/linux-process-management.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 294.192666,
          "tokens": 5088
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/tls-ssl.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/tls-ssl.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 668.428417,
          "tokens": 4693
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/container-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 482.056875,
          "tokens": 5492
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 440.357333,
          "tokens": 8234
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/infrastructure-as-code.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 0.3333333333333333,
          "ndcg_5": 0.5,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 583.215583,
          "tokens": 1587
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/scale_docs/cloud-storage-options.md",
            "tests/fixtures/scale_docs/load-balancing.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 367.702666,
          "tokens": 5736
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 401.238292,
          "tokens": 4253
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/network-security-hardening.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 336.34925,
          "tokens": 3400
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/nosql-patterns.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 364.870958,
          "tokens": 6625
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 268.035041,
          "tokens": 8460
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/cdn-architecture.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/dns-resolution.md",
            "tests/fixtures/scale_docs/cdn-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 275.941208,
          "tokens": 6072
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 447.04275,
          "tokens": 2685
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/connection-pooling.md",
            "tests/fixtures/scale_docs/testing-strategies.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 545.998709,
          "tokens": 1883
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.43067655807339306,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 351.865917,
          "tokens": 8216
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/event-driven-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 587.464,
          "tokens": 2349
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/dependency-injection.md",
            "tests/fixtures/scale_docs/container-fundamentals.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 331.13125,
          "tokens": 11411
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/design-patterns-creational.md",
            "tests/fixtures/scale_docs/network-security-hardening.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/scale_docs/compliance-frameworks.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 458.566542,
          "tokens": 1696
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 531.471292,
          "tokens": 1775
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/penetration-testing-methodology.md",
            "tests/fixtures/scale_docs/serverless-architecture.md",
            "tests/fixtures/scale_docs/chaos-engineering.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 607.608083,
          "tokens": 2320
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/functional-programming.md",
            "tests/fixtures/scale_docs/disaster-recovery.md",
            "tests/fixtures/scale_docs/data-quality.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/scale_docs/error-handling-patterns.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 416.849125,
          "tokens": 7659
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/vpn-tunneling.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 369.074708,
          "tokens": 979
        },
        {
          "competitor": "candlekeep-hybrid",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/stream-processing.md",
            "tests/fixtures/scale_docs/cost-optimization.md",
            "tests/fixtures/scale_docs/testing-strategies.md",
            "tests/fixtures/scale_docs/stream-processing.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 601.970042,
          "tokens": 3333
        }
      ]
    },
    "ablation-no-bardic": {
      "summary": {
        "n": 30,
        "mrr": 0.64,
        "mrr_ci": [
          0.4731666666666668,
          0.8066666666666666
        ],
        "hit_rate_5": 0.8666666666666667,
        "hit_rate_5_ci": [
          0.7333333333333333,
          0.9666666666666667
        ],
        "hit_rate_1": 0.8333333333333334,
        "precision_5": 0.5738888888888888,
        "latency_p50": 134.41425,
        "avg_tokens": 399.8666666666667,
        "by_category": {
          "context-dependent": {
            "mrr": 0.5,
            "hit_rate_5": 0.5,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.7000000000000001,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 1.0,
            "hit_rate_5": 1.0,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 137.043667,
          "tokens": 476
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/scale_docs/load-balancing.md",
            "tests/fixtures/scale_docs/nosql-patterns.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 160.194459,
          "tokens": 614
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 74.11825,
          "tokens": 6
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 679.111625,
          "tokens": 212
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 100.950166,
          "tokens": 123
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 62.645667,
          "tokens": 197
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 274.303125,
          "tokens": 1721
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/penetration-testing-methodology.md",
            "tests/fixtures/scale_docs/linux-performance-tuning.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 170.730583,
          "tokens": 302
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/postgresql-internals.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 66.905875,
          "tokens": 391
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/tcp-ip-fundamentals.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/linux-filesystem.md",
            "tests/fixtures/scale_docs/service-mesh.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 353.3505,
          "tokens": 731
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/scale_docs/linux-networking-config.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/data-lake-architecture.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md"
          ],
          "rr": 0.2,
          "ndcg_5": 0.38685280723454163,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 652.08825,
          "tokens": 534
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/penetration-testing-methodology.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 464.859333,
          "tokens": 332
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 699.48775,
          "tokens": 910
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 63.674041,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/data-pipeline-design.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-serialization.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 320.823167,
          "tokens": 753
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/grpc-protocol.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/linux-networking-config.md",
            "tests/fixtures/scale_docs/cost-optimization.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 744.084209,
          "tokens": 920
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/document-processing.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 134.41425,
          "tokens": 734
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/multi-region-deployment.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 151.478125,
          "tokens": 1196
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6666666666666666,
          "latency_ms": 131.603208,
          "tokens": 416
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 89.4625,
          "tokens": 67
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/scale_docs/data-serialization.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 113.788459,
          "tokens": 76
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/testing-strategies.md",
            "tests/fixtures/scale_docs/shell-scripting.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 667.750375,
          "tokens": 304
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/scale_docs/migration-strategies.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.25,
          "latency_ms": 182.085416,
          "tokens": 891
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/container-fundamentals.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 70.31425,
          "tokens": 90
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 92.534417,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 91.326708,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 101.763667,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 92.264625,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 103.324833,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-bardic-knowledge",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 83.619,
          "tokens": 0
        }
      ]
    },
    "ablation-no-arcane": {
      "summary": {
        "n": 30,
        "mrr": 0.5916666666666667,
        "mrr_ci": [
          0.425,
          0.7583333333333333
        ],
        "hit_rate_5": 0.8333333333333334,
        "hit_rate_5_ci": [
          0.6991666666666674,
          0.9333333333333333
        ],
        "hit_rate_1": 0.7666666666666667,
        "precision_5": 0.6433333333333333,
        "latency_p50": 76.281833,
        "avg_tokens": 311.06666666666666,
        "by_category": {
          "context-dependent": {
            "mrr": 0.6666666666666666,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          },
          "exact-identifier": {
            "mrr": 0.7083333333333334,
            "hit_rate_5": 0.8333333333333334,
            "n": 6
          },
          "negation": {
            "mrr": 0.9166666666666666,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "out-of-domain": {
            "mrr": 0.0,
            "hit_rate_5": 1.0,
            "n": 6
          },
          "redundancy": {
            "mrr": 0.6666666666666666,
            "hit_rate_5": 0.6666666666666666,
            "n": 6
          }
        }
      },
      "details": [
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "authentication methods without using passwords",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/secrets-management.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 0.5,
          "ndcg_5": 0.6309297535714575,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 84.778375,
          "tokens": 341
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "caching strategies not involving Redis",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/database-sharding.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 265.559042,
          "tokens": 426
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "database design without normalization",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/database-design.md",
            "tests/fixtures/sample_docs/database-design.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 76.281833,
          "tokens": 385
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "API design excluding REST",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/api-design.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 174.50925,
          "tokens": 437
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "microservices communication except synchronous",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 57.681041,
          "tokens": 242
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "search techniques without keyword matching",
          "category": "negation",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 62.236042,
          "tokens": 277
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "Elasticsearch 8.x breaking changes",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md",
            "tests/fixtures/scale_docs/elasticsearch-basics.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 80.245708,
          "tokens": 552
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "Redis 7.0 AOF rewrite",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md",
            "tests/fixtures/scale_docs/redis-data-structures.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 59.346791,
          "tokens": 436
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "PostgreSQL 15 new features",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 13.326084,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "TLS 1.3 zero round trip",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/service-mesh.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/zero-trust-architecture.md",
            "tests/fixtures/scale_docs/tls-ssl.md",
            "tests/fixtures/scale_docs/vpn-tunneling.md"
          ],
          "rr": 0.25,
          "ndcg_5": 0.43067655807339306,
          "hit_rate_1": 0.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 301.324708,
          "tokens": 530
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "Kubernetes v1.24 dockershim removal",
          "category": "exact-identifier",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/kubernetes-architecture.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/container-security.md",
            "tests/fixtures/scale_docs/kubernetes-architecture.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.5,
          "latency_ms": 94.962791,
          "tokens": 540
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "OWASP Top 10 2021 vulnerabilities",
          "category": "exact-identifier",
          "difficulty": "medium",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md",
            "tests/fixtures/scale_docs/security-audit-checklist.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 159.422458,
          "tokens": 728
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "How do you store and refresh JWT tokens securely?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/scale_docs/owasp-top-10.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.4,
          "latency_ms": 269.077042,
          "tokens": 534
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "What happens when a cache entry expires and multiple requests arrive simultaneously?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.139167,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "How do you handle schema migrations in a microservices architecture?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/database-migrations.md",
            "tests/fixtures/scale_docs/migration-strategies.md",
            "tests/fixtures/scale_docs/data-pipeline-design.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.2,
          "latency_ms": 214.012708,
          "tokens": 754
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "What are the steps to implement rate limiting on an API endpoint?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/network-security-hardening.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/scale_docs/api-gateway-patterns.md",
            "tests/fixtures/scale_docs/network-security-hardening.md"
          ],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 182.534458,
          "tokens": 751
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "How does the embedding model affect search quality in a vector database?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md",
            "tests/fixtures/sample_docs/semantic-search.md",
            "tests/fixtures/sample_docs/vector-databases.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 58.546875,
          "tokens": 295
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "What is the complete process for setting up database replication?",
          "category": "context-dependent",
          "difficulty": "hard",
          "retrieved_sources": [
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md",
            "tests/fixtures/scale_docs/database-replication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 87.57325,
          "tokens": 731
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "caching",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/scale_docs/ci-cd-pipelines.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md",
            "tests/fixtures/sample_docs/caching.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 127.103,
          "tokens": 367
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "authentication",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md",
            "tests/fixtures/sample_docs/authentication.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 24.890167,
          "tokens": 113
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "database",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 14.921541,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "API",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/sample_docs/api-design.md",
            "tests/fixtures/scale_docs/api-security.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.6,
          "latency_ms": 253.043334,
          "tokens": 484
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "microservices",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/scale_docs/data-warehouse-modeling.md",
            "tests/fixtures/sample_docs/microservices.md",
            "tests/fixtures/sample_docs/microservices.md"
          ],
          "rr": 1.0,
          "ndcg_5": 1.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 0.8,
          "latency_ms": 146.039625,
          "tokens": 409
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "containers",
          "category": "redundancy",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 0.0,
          "hit_rate_5": 0.0,
          "precision_5": 0.0,
          "latency_ms": 11.752292,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "how to train a neural network from scratch",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.418375,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "best practices for raising chickens in urban areas",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.625542,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "history of the Roman Empire's decline",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.861,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "how to solve differential equations numerically",
          "category": "out-of-domain",
          "difficulty": "medium",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.117083,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "recipe for authentic Neapolitan pizza dough",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.981875,
          "tokens": 0
        },
        {
          "competitor": "ablation-no-arcane-recall",
          "corpus": "diagnostic",
          "query": "techniques for oil painting landscapes",
          "category": "out-of-domain",
          "difficulty": "easy",
          "retrieved_sources": [],
          "rr": 0.0,
          "ndcg_5": 0.0,
          "hit_rate_1": 1.0,
          "hit_rate_5": 1.0,
          "precision_5": 1.0,
          "latency_ms": 10.95475,
          "tokens": 0
        }
      ]
    }
  }
}