{
  "$schema": "https://evalytic.dev/schemas/bench-report-v1.json",
  "version": "1.0",
  "name": "bench-dbb26792",
  "created_at": "2026-02-27T10:29:58.539855+00:00",
  "duration_seconds": 28.5,
  "config": {
    "models": [
      "flux-schnell",
      "flux-dev",
      "sdxl"
    ],
    "judge": "gemini-2.5-flash",
    "dimensions": [
      "visual_quality",
      "prompt_adherence",
      "text_rendering"
    ],
    "metrics": [],
    "pipeline": "text2img",
    "seed": null,
    "image_size": null,
    "concurrency": 4
  },
  "items": [
    {
      "item_id": "item-001",
      "prompt": "A neon sign reading 'MIDNIGHT CAFE' above a door in a rainy Tokyo alley at night, reflections on wet pavement",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a902501/e0lz8W76ge_4C-l60QMzo.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-dbb26792/flux-schnell/item-001.jpg",
          "generation_time_ms": 1153,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 0.85,
              "explanation": "The image features a strong atmospheric composition with effective lighting and reflections, creating a compelling mood. However, its quality is significantly hampered by prominent text artifacts, including misspelled words and unreadable gibberish on multiple signs.",
              "evidence": [
                "The poster on the door clearly displays 'COREITY CAIFE' instead of 'COFFEE CAFE' or similar, and the text below it is nonsensical.",
                "Background signs on the left and reflections in the window on the right contain distorted and unreadable characters, typical of AI-generated text artifacts.",
                "The main 'MIDNIGHT CAFE' neon sign, while mostly clear, shows a slight lack of crispness in the 'CAFE' portion compared to 'MIDNIGHT'."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 0.9,
              "explanation": "The image perfectly captures all aspects of the prompt, featuring a clear 'MIDNIGHT CAFE' neon sign above a door in a dark, wet alley. The reflections on the pavement are well-rendered, enhancing the rainy night atmosphere and evoking the specified location.",
              "evidence": [
                "The neon sign clearly reads 'MIDNIGHT CAFE' and is positioned above a door.",
                "The scene is an alley at night, with dark surroundings and prominent reflections on the wet pavement, indicating rain.",
                "The overall aesthetic, including the narrow alley and neon lights, strongly suggests a Tokyo-like environment."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The expected text 'MIDNIGHT CAFE' on the prominent neon sign is perfectly rendered, crisp, and accurate. Although there is other garbled text on secondary signs and posters, it was not part of the text explicitly expected by the prompt.",
              "evidence": [
                "The neon sign clearly and accurately displays 'MIDNIGHT CAFE'.",
                "The letters of 'MIDNIGHT CAFE' are well-formed, legible, and free of artifacts."
              ]
            }
          },
          "metrics": {},
          "overall_score": 4.333333333333333,
          "status": "success"
        },
        "flux-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a902501/rI-BV0K6fU0WWICJ2fKCV.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-dbb26792/flux-dev/item-001.jpg",
          "generation_time_ms": 2558,
          "generation_cost_usd": 0.025,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 0.85,
              "explanation": "The image successfully creates a moody, atmospheric scene with good lighting and reflections. However, there are noticeable artifacts in the text on the sign, which is a prominent feature, and the person inside the cafe lacks detail and appears blurry.",
              "evidence": [
                "The repeated 'CAF\u00c9' text on the sign is distorted and partially illegible, appearing as 'CAF\u00c9 CAF\u00c9)'.",
                "The person visible inside the cafe is blurry and lacks distinct features, typical of AI generation artifacts."
              ]
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly adheres to the prompt, capturing the requested scene with high fidelity. The primary deviation is the additional, slightly garbled text on the neon sign below the main 'MIDNIGHT CAFE'.",
              "evidence": [
                "The neon sign clearly reads 'MIDNIGHT CAFE' and is positioned above a door.",
                "The scene depicts a rainy night in what appears to be a Tokyo-style alley, complete with prominent reflections on the wet pavement.",
                "The sign also includes additional, repetitive text 'CAF\u00c9 CAFE)' below the main 'MIDNIGHT CAFE', which was not specified in the prompt and includes a minor artifact."
              ]
            },
            "text_rendering": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "The primary text 'MIDNIGHT CAFE' is clearly and correctly rendered on the neon sign. However, directly below it, the sign incorrectly displays 'CAF\u00c9 CAFE)', which was not part of the expected text and includes a repetition and a closing parenthesis artifact.",
              "evidence": [
                "The main neon sign accurately reads 'MIDNIGHT CAFE'.",
                "An additional line of text below the main sign incorrectly displays 'CAF\u00c9 CAFE)'."
              ]
            }
          },
          "metrics": {},
          "overall_score": 3.3333333333333335,
          "status": "success"
        },
        "sdxl": {
          "image_url": "https://v3b.fal.media/files/b/0a902501/WMqaV8USdKn-MttGbKxET.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-dbb26792/sdxl/item-001.jpg",
          "generation_time_ms": 2573,
          "generation_cost_usd": 0.01,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 0.85,
              "explanation": "The image presents a strong aesthetic with atmospheric lighting and reflections, but it is significantly marred by prominent text artifacts. While the overall composition is good, the illegible and malformed text elements detract from the realism and quality.",
              "evidence": [
                "The characters on the main neon sign are malformed and not perfectly coherent, particularly the character on the far right.",
                "The smaller blue characters below the main neon sign are indistinct and appear as garbled strokes.",
                "The text on the red posters affixed to the green door is completely illegible gibberish, a common AI artifact."
              ]
            },
            "prompt_adherence": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "The image successfully captures the atmosphere of a rainy Tokyo alley at night with reflections on wet pavement and a neon sign above a door. However, a crucial detail, the specific text 'MIDNIGHT CAFE' on the neon sign, is not present, instead showing Japanese characters.",
              "evidence": [
                "The image features a prominent neon sign above a door, a rainy atmosphere with wet pavement and clear reflections, and a night setting in an alley that strongly resembles Tokyo.",
                "The neon sign does not read 'MIDNIGHT CAFE'; it displays characters that appear to be Japanese, which is a direct mismatch to the requested text."
              ]
            },
            "text_rendering": {
              "score": 1.0,
              "confidence": 1.0,
              "explanation": "The image completely fails to render the expected English text 'MIDNIGHT CAFE'. Instead, the prominent neon sign displays unreadable Asian characters that do not correspond to the prompt's request.",
              "evidence": [
                "The main neon sign clearly shows Asian characters, not English letters.",
                "The expected text 'MIDNIGHT CAFE' is entirely absent from the image.",
                "The characters themselves, while recognizable as text, are not perfectly formed and show some distortion."
              ]
            }
          },
          "metrics": {},
          "overall_score": 2.3333333333333335,
          "status": "success"
        }
      }
    }
  ],
  "summary": {
    "flux-schnell": {
      "overall_score": 4.33,
      "dimension_averages": {
        "visual_quality": 3.0,
        "prompt_adherence": 5.0,
        "text_rendering": 5.0
      },
      "dimension_confidence": {
        "visual_quality": 0.85,
        "prompt_adherence": 0.9,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.92,
      "metric_averages": {},
      "total_generation_cost_usd": 0.003,
      "total_generation_time_ms": 1153,
      "cost_per_image": 0.003,
      "score_per_dollar": 1443.3,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.0
      },
      "overall_stddev": 0.94,
      "sample_count": 1,
      "total_items": 1,
      "success_rate": 1.0
    },
    "flux-dev": {
      "overall_score": 3.33,
      "dimension_averages": {
        "visual_quality": 3.0,
        "prompt_adherence": 4.0,
        "text_rendering": 3.0
      },
      "dimension_confidence": {
        "visual_quality": 0.85,
        "prompt_adherence": 0.9,
        "text_rendering": 0.9
      },
      "avg_confidence": 0.88,
      "metric_averages": {},
      "total_generation_cost_usd": 0.025,
      "total_generation_time_ms": 2558,
      "cost_per_image": 0.025,
      "score_per_dollar": 133.2,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.0
      },
      "overall_stddev": 0.47,
      "sample_count": 1,
      "total_items": 1,
      "success_rate": 1.0
    },
    "sdxl": {
      "overall_score": 2.33,
      "dimension_averages": {
        "visual_quality": 3.0,
        "prompt_adherence": 3.0,
        "text_rendering": 1.0
      },
      "dimension_confidence": {
        "visual_quality": 0.85,
        "prompt_adherence": 0.9,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.92,
      "metric_averages": {},
      "total_generation_cost_usd": 0.01,
      "total_generation_time_ms": 2573,
      "cost_per_image": 0.01,
      "score_per_dollar": 233.0,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.0
      },
      "overall_stddev": 0.94,
      "sample_count": 1,
      "total_items": 1,
      "success_rate": 1.0
    }
  },
  "ranking": [
    [
      "flux-schnell",
      4.33
    ],
    [
      "flux-dev",
      3.33
    ],
    [
      "sdxl",
      2.33
    ]
  ],
  "efficiency_ranking": [
    [
      "flux-schnell",
      1443.3
    ],
    [
      "sdxl",
      233.0
    ],
    [
      "flux-dev",
      133.2
    ]
  ],
  "winner": "flux-schnell",
  "best_value": "flux-schnell",
  "correlation": {},
  "cost": {
    "generation": {
      "total_usd": 0.038,
      "by_model": {
        "flux-schnell": 0.003,
        "flux-dev": 0.025,
        "sdxl": 0.01
      },
      "request_count": 3
    },
    "judge": {
      "total_usd": 0.0045,
      "provider": "gemini-2.5-flash",
      "request_count": 9
    },
    "metrics": {
      "total_usd": 0.0,
      "note": "Local computation"
    },
    "total_usd": 0.0425
  },
  "metadata": {
    "evalytic_version": "0.2.1",
    "python_version": "3.14.3",
    "platform": "darwin-arm64"
  }
}