{
  "$schema": "https://evalytic.ai/schemas/bench-report-v1.json",
  "version": "1.0",
  "name": "leaderboard-test-3x5",
  "created_at": "2026-03-12T10:46:21.318412+00:00",
  "duration_seconds": 135.7,
  "config": {
    "models": [
      "flux-schnell",
      "flux-2-dev",
      "z-image-turbo",
      "ideogram-v3",
      "nano-banana"
    ],
    "judge": "gemini-2.5-flash",
    "dimensions": [
      "visual_quality",
      "prompt_adherence",
      "text_rendering"
    ],
    "metrics": [
      "clip",
      "sharpness",
      "aesthetic"
    ],
    "pipeline": "text2img",
    "seed": 42,
    "image_size": null,
    "concurrency": 8,
    "scoring_config": {
      "clip_score": {
        "flag_threshold": 0.18,
        "weight": 0.2,
        "normalize_range": [
          0.18,
          0.35
        ]
      },
      "lpips": {
        "flag_threshold": 0.4,
        "weight": 0.2,
        "normalize_range": [
          0.4,
          0.95
        ]
      },
      "face_similarity": {
        "flag_threshold": 0.6,
        "weight": 0.2,
        "normalize_range": [
          0.6,
          0.95
        ]
      },
      "aesthetic_score": {
        "flag_threshold": 0.3,
        "weight": 0.15,
        "normalize_range": [
          0.3,
          0.8
        ]
      }
    }
  },
  "items": [
    {
      "item_id": "item-001",
      "prompt": "A glowing neon sign reading EVALYTIC in a dark rainy alley, reflections on wet cobblestones, cinematic lighting",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/QnHxHegMaDrY2J0DySxRA.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-schnell/item-001.jpg",
          "generation_time_ms": 1417,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image exhibits high visual quality with excellent atmospheric rendering, particularly the neon sign and its reflections on the wet ground. However, there is a minor artifact in the form of illegible text on a sign on the right-hand building.",
              "evidence": [
                "Sharp and coherent rendering of the 'EVALYTIC' neon sign and its realistic reflections.",
                "Effective use of lighting and color contrast to create a strong, moody atmosphere.",
                "Presence of illegible and nonsensical text ('SEPET SULANTICC') on a sign on the right-hand building."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt, featuring a clearly legible neon sign 'EVALYTIC' in a dark, rainy alley. The reflections on the wet cobblestones are prominent and well-rendered, contributing to the overall cinematic lighting and atmosphere.",
              "evidence": [
                "The neon sign 'EVALYTIC' is perfectly rendered and glowing.",
                "The setting is clearly a dark, rainy alley with visible rain effects.",
                "Strong, accurate reflections of the neon sign are present on the wet cobblestones.",
                "The lighting is dramatic and atmospheric, consistent with 'cinematic lighting'."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered on the neon sign. Each letter is crisp, correctly formed, and highly legible, matching the expected content precisely.",
              "evidence": [
                "The neon sign clearly displays 'EVALYTIC'.",
                "All letters are correctly spelled and well-defined.",
                "There are no rendering issues or artifacts affecting readability."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3127,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9446,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4613,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 4.666666666666667,
          "status": "success"
        },
        "flux-2-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/VySpLiihUV_TnlFd2WPui_tfV2k6iI.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-2-dev/item-001.png",
          "generation_time_ms": 3285,
          "generation_cost_usd": 0.012,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image demonstrates professional quality with sharp, coherent rendering of the neon sign and highly realistic reflections on the wet cobblestone street. The atmospheric rain and moody lighting are consistently well-executed, creating a compelling scene without any visible artifacts or distortions.",
              "evidence": [
                "The neon text 'EVALYTIC' is perfectly legible, sharply rendered, and exhibits a realistic glow with a smooth color gradient.",
                "The reflections of the neon sign on the wet cobblestone street are vibrant, accurate, and contribute significantly to the image's realism and atmosphere.",
                "The rain effect and wet surfaces are consistently applied and visually convincing throughout the scene."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt, featuring a clear neon sign with the specified text, set in a dark, rainy alley with excellent reflections on wet cobblestones and a strong cinematic atmosphere.",
              "evidence": [
                "The neon sign clearly reads 'EVALYTIC' and is glowing.",
                "The setting is a dark, rainy alley with visible rain streaks and a wet environment.",
                "Vibrant reflections of the neon sign's colors are present on the wet cobblestone ground.",
                "The dramatic lighting and moody atmosphere align with 'cinematic lighting'."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered as a neon sign. Each letter is clear, correctly spelled, and the neon effect is well-executed with appropriate glow and reflections.",
              "evidence": [
                "The word 'EVALYTIC' is spelled correctly.",
                "Each letter is distinct and easily readable.",
                "The neon glow effect is consistent and visually appealing."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3409,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8577,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.5005,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "z-image-turbo": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/4YG2056EIX_9SMb4YRcxT_fRhF1Ywa.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/z-image-turbo/item-001.png",
          "generation_time_ms": 2217,
          "generation_cost_usd": 0.005,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality with no discernible artifacts or inconsistencies. The lighting, reflections, and atmospheric effects are rendered with professional precision, creating a highly coherent and immersive scene.",
              "evidence": [
                "The neon text 'EVALYTIC' is perfectly legible and sharply rendered, a common challenge for AI.",
                "Realistic and detailed reflections of the neon light are visible on the wet cobblestone street.",
                "The rain streaks are natural-looking and consistently applied, enhancing the moody atmosphere.",
                "Textures on the brick walls and metal shutters are sharp and well-defined."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image is an excellent representation of the prompt, perfectly capturing every detail requested. The neon sign, rainy alley, wet cobblestone reflections, and cinematic lighting are all accurately and beautifully rendered.",
              "evidence": [
                "The image features a glowing neon sign that accurately reads 'EVALYTIC'.",
                "The setting is clearly a dark, rainy alley with visible rain streaks.",
                "There are prominent and realistic reflections of the neon light on the wet cobblestones.",
                "The lighting creates a dramatic and atmospheric 'cinematic' feel."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered as a glowing neon sign. Each letter is crisp, correctly spelled, and highly legible, with no artifacts or distortions.",
              "evidence": [
                "The word 'EVALYTIC' is spelled correctly.",
                "Each letter is clearly defined and readable.",
                "The neon glow effect is well-executed and enhances readability."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3547,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.944,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4901,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc1f/0vJcj1Uk8cVStgSIwxVQb_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/ideogram-v3/item-001.png",
          "generation_time_ms": 57167,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality with sharp details, realistic lighting, and accurate reflections. The neon sign is perfectly rendered with a convincing glow, and the wet ground accurately mirrors the scene's elements without artifacts.",
              "evidence": [
                "Sharp and perfectly formed neon text 'EVALYTIC' with a realistic glow.",
                "Accurate and detailed reflections of the sign and street lamp in the wet ground, showing appropriate distortion from water ripples.",
                "Consistent and atmospheric lighting from both the neon sign and the street lamp, illuminating the brick walls and ground realistically."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image perfectly matches all elements of the prompt. It features a clear neon sign reading 'EVALYTIC' in a dark alley, with excellent reflections on wet cobblestones, and a strong sense of cinematic lighting.",
              "evidence": [
                "The neon sign clearly displays 'EVALYTIC' as requested.",
                "The setting is a dark alley with wet, reflective cobblestones showing the sign's reflection.",
                "The lighting is dramatic and atmospheric, consistent with 'cinematic lighting'."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered on the neon sign. Each letter is crisp, correctly formed, and clearly legible, including its reflection on the wet ground.",
              "evidence": [
                "The word 'EVALYTIC' is spelled correctly on the sign.",
                "Each letter is distinct and well-formed with a consistent neon glow.",
                "The reflection of the text on the wet cobblestones is also accurate and readable."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3036,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9552,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4452,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "nano-banana": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc1e/2zbXrkbAqRN_yF_U1Nsoe_Ty2UcQsH.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/nano-banana/item-001.png",
          "generation_time_ms": 53802,
          "generation_cost_usd": 0.0398,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image exhibits excellent visual quality with professional-grade rendering. All elements, from the sharp neon sign and realistic rain to the detailed wet ground reflections and atmospheric mist, are rendered flawlessly without any visible artifacts or inconsistencies.",
              "evidence": [
                "Sharp and consistent rendering of the neon sign and its glow.",
                "Realistic rain streaks and accurate reflections on the wet pavement.",
                "Detailed textures on the brick wall, trash cans, and graffiti.",
                "Cohesive atmospheric effects including mist and dynamic lighting."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt. It features a glowing neon sign reading 'EVALYTIC' in a dark, rainy alley, with excellent reflections on the wet ground and a strong sense of cinematic lighting.",
              "evidence": [
                "The neon sign 'EVALYTIC' is clearly visible and glowing.",
                "The setting is a dark alley with visible rain and wet surfaces.",
                "Reflections of the neon sign are prominent on the wet cobblestones/pavement.",
                "The lighting is dramatic and atmospheric, consistent with 'cinematic lighting'."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered as a neon sign. It is crisp, clearly legible, and accurately spelled, with excellent reflections on the wet ground.",
              "evidence": [
                "The word 'EVALYTIC' is clearly visible and correctly spelled on the neon sign.",
                "The individual letters are well-formed and distinct, even with the neon glow effect.",
                "The reflections of the text on the wet cobblestones are also accurate and readable."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3206,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8651,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4912,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        }
      }
    },
    {
      "item_id": "item-002",
      "prompt": "A black cat sitting on a stack of old books in a cozy library, warm golden light from a desk lamp, dust particles in the air",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/EYzd2NQcG7QSq8SNOR-qq.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-schnell/item-002.jpg",
          "generation_time_ms": 1137,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 4.0,
              "confidence": 0.8,
              "explanation": "The image exhibits good visual quality with a well-rendered main subject and atmospheric lighting. The black cat and the stack of books it sits on are detailed and realistic, contributing to a strong foreground focus. However, the background bookshelves are noticeably blurry and lack distinct book details, appearing somewhat like a textured backdrop rather than individual books.",
              "evidence": [
                "The black cat is rendered with good fur texture, realistic eyes, and natural posture.",
                "The stacked books in the foreground have convincing textures and appear stable.",
                "The background bookshelves are highly blurred and lack individual book definition, appearing indistinct.",
                "The light source within the lamp appears somewhat flat and artificial, rather than a glowing bulb."
              ]
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly matches most aspects of the prompt, featuring a black cat on old books in a library with warm golden light from a lamp. The only minor deviation is the absence of clearly visible 'dust particles in the air'.",
              "evidence": [
                "A black cat is prominently featured.",
                "The cat is sitting on a stack of old-looking books.",
                "The setting is clearly a library with bookshelves in the background.",
                "A desk lamp emits warm golden light, illuminating the scene.",
                "Distinct dust particles in the air are not visibly rendered."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in the image based on the prompt, and no text is present in the generated image. The image successfully avoids generating any garbled or incorrect text.",
              "evidence": [
                "No text is visible on the books, lamp, or any other element in the image.",
                "The prompt did not specify any text content to be rendered."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3366,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.865,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.5734,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 4.333333333333333,
          "status": "success"
        },
        "flux-2-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/AqFzD5eEA6IQfJp3e9hFi_GYHe7zyM.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-2-dev/item-002.png",
          "generation_time_ms": 3319,
          "generation_cost_usd": 0.012,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.9,
              "explanation": "The image exhibits excellent visual quality with sharp details on the main subjects (cat, lamp, foreground books) and a well-executed depth of field. The dramatic and realistic lighting creates a strong atmosphere without any noticeable artifacts or inconsistencies.",
              "evidence": [
                "Sharp and detailed rendering of the cat's fur and eyes, particularly visible in the illuminated areas.",
                "Realistic metallic texture of the lamp and a convincing glowing light source with atmospheric dust motes.",
                "Well-defined aged textures on the books in the foreground, contributing to the overall realism."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures every element described in the prompt. The black cat, stack of old books, cozy library setting, warm golden light from the lamp, and dust particles are all clearly and accurately depicted.",
              "evidence": [
                "A black cat is prominently featured.",
                "The cat is sitting on a stack of old, worn books.",
                "The background shows bookshelves, establishing a library setting.",
                "A desk lamp emits warm golden light, illuminating the scene.",
                "Visible dust particles are present in the light beam from the lamp."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in the image based on the provided prompt, and indeed, no text is present in the generated image. The image accurately reflects the absence of text.",
              "evidence": [
                "No text is visible anywhere in the image.",
                "The prompt 'A black cat sitting on a stack of old books in a cozy library, warm golden light from a desk lamp, dust particles in the air' does not specify any text to be rendered."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3561,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8551,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.5742,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "z-image-turbo": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/HruOuzp-6XSIMg8DLQ85p_eYWjit7b.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/z-image-turbo/item-002.png",
          "generation_time_ms": 2471,
          "generation_cost_usd": 0.005,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits professional-level quality with excellent atmospheric lighting, realistic textures, and a well-executed depth of field. The subtle details, such as the illuminated dust particles and the soft fur of the cat, significantly enhance the realism and overall aesthetic.",
              "evidence": [
                "Realistic fur texture and expressive yellow eyes of the black cat, especially where illuminated by the lamp.",
                "Excellent atmospheric lighting from the desk lamp, highlighting dust particles in the air and creating a warm, inviting glow.",
                "Sharp focus on the foreground elements (cat, lamp, books) with a natural and pleasing bokeh effect in the background bookshelves."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements described in the prompt, including the black cat, old books, cozy library setting, warm golden light from a desk lamp, and visible dust particles in the air. Every detail is accurately represented.",
              "evidence": [
                "A black cat is prominently featured and is the main subject.",
                "The cat is sitting on a stack of old, worn books.",
                "The background clearly depicts a library with bookshelves filled with books.",
                "A desk lamp provides warm golden light, illuminating the scene.",
                "Numerous dust particles are visible floating in the light beam from the lamp."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected to be rendered in this image based on the provided prompt, which describes a scene rather than specific text content. The image successfully depicts the scene without any text elements.",
              "evidence": [
                "No text is present anywhere in the image.",
                "The prompt describes visual elements and a scene, not text to be displayed."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3278,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.7843,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.5808,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc1f/nvsE9tZvkV--PvHFjP_It_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/ideogram-v3/item-002.png",
          "generation_time_ms": 56379,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.9,
              "explanation": "The image demonstrates excellent visual quality with sharp details, realistic textures, and atmospheric lighting. There are no visible artifacts or anatomical inconsistencies, and the depth of field is well-executed, contributing to a professional-quality render.",
              "evidence": [
                "Sharp and realistic rendering of the cat's fur, eyes, and whiskers.",
                "Well-defined textures on the books and the lamp, showing appropriate wear and detail.",
                "Atmospheric lighting with natural-looking sunbeams and dust motes, enhancing the scene without appearing artificial.",
                "Absence of noticeable artifacts, distortions, or incoherent elements across the image."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt. A black cat is sitting on a stack of old books in a library, illuminated by warm golden light from a desk lamp, with visible dust particles in the air.",
              "evidence": [
                "A black cat is prominently featured.",
                "The cat is sitting on a stack of old, worn books.",
                "The background clearly depicts a library with bookshelves.",
                "A desk lamp provides warm, golden light, backlighting the cat.",
                "Visible light rays with illuminated dust particles are present on the left side."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in the image based on the provided prompt, and indeed, no text is present. The image successfully avoids generating any garbled or unintended text.",
              "evidence": [
                "No text is visible anywhere in the image.",
                "The prompt 'A black cat sitting on a stack of old books in a cozy library, warm golden light from a desk lamp, dust particles in the air' does not specify any text to be rendered."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3369,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8983,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.6036,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        },
        "nano-banana": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/t7ute8XNDbNKM25Ap7dBJ_79ZNgCHa.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/nano-banana/item-002.png",
          "generation_time_ms": 7046,
          "generation_cost_usd": 0.0398,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.9,
              "explanation": "The image exhibits excellent visual quality with sharp details, realistic textures, and masterful lighting. The composition is well-balanced, and elements like the cat's fur and the metallic lamp are rendered with high fidelity, creating a coherent and appealing scene.",
              "evidence": [
                "Sharp rendering of the cat's fur and reflective eyes.",
                "Realistic metallic texture and light emission from the desk lamp.",
                "Consistent and atmospheric lighting, creating a cozy ambiance with warm glows from the lamp and fireplace.",
                "Detailed textures on the foreground books and wooden desk, with only minor illegibility in background text."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly matches every detail specified in the prompt. All elements, from the subject to the setting and atmospheric effects, are accurately represented.",
              "evidence": [
                "A black cat is clearly depicted sitting on a stack of old books.",
                "The setting is a cozy library with bookshelves and a fireplace.",
                "Warm golden light emanates from a desk lamp, illuminating the scene.",
                "Visible dust particles are present in the air, catching the light."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The prompt did not specify any text to be included in the image. While there are open books with patterns resembling text, these are not intended to be readable or convey specific information, thus no text rendering quality issues are present.",
              "evidence": [
                "No specific text content was requested in the prompt.",
                "The patterns on the open book pages are abstract and not meant to be legible text."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3298,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.7948,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.611,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        }
      }
    },
    {
      "item_id": "item-003",
      "prompt": "A vintage coffee shop storefront with hand-lettered chalkboard menu, morning sunlight, 35mm film photography style",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/hrHseqXoq2Fr2JKIZqiaL.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-schnell/item-003.jpg",
          "generation_time_ms": 1146,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 2.0,
              "confidence": 0.9,
              "explanation": "The image suffers from significant text incoherence, with all prominent signs and menus displaying garbled and unreadable words. While the overall composition, lighting, and general object rendering are acceptable, the failure to render legible text is a major quality issue.",
              "evidence": [
                "The main sign reads 'Coffeee Chop' with an extra 'e' in 'Coffeee' and 'Chop' is slightly distorted.",
                "The menu board on the right contains completely incoherent and unreadable text, such as 'Choces Fithe Rogie' and 'Scnord Caffing'.",
                "The text on the window and the list below it are also garbled and nonsensical, like 'Cuffeehad's' and 'Pacion Cedenting'."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image perfectly captures all aspects of the prompt, featuring a vintage-style coffee shop storefront with clear morning sunlight and a hand-lettered chalkboard menu. The overall aesthetic strongly evokes a 35mm film photography style.",
              "evidence": [
                "The storefront design and 'Coffeee Chop' signage convey a vintage coffee shop feel.",
                "A prominent hand-lettered chalkboard menu is visible on the right, and another hand-lettered menu is inside the window.",
                "The lighting exhibits distinct long shadows and a warm glow, indicative of morning sunlight.",
                "The image possesses a natural, slightly muted color palette and depth consistent with 35mm film photography."
              ]
            },
            "text_rendering": {
              "score": 1.0,
              "confidence": 1.0,
              "explanation": "The text throughout the image is largely unreadable, garbled, or contains significant misspellings. The main sign 'Coffeee Chop' is misspelled, and the extensive text on both chalkboard menus is nonsensical gibberish, failing to convey any coherent information.",
              "evidence": [
                "The main sign reads 'Coffeee Chop' with 'Coffeee' being misspelled and 'Chop' being an unusual choice for a coffee shop.",
                "The text 'MONING V' visible inside the window is garbled and incomplete.",
                "Both chalkboard menus are filled with unreadable gibberish, such as 'Cuffeehad's', 'Pacion Cedenting', 'Choces Fithe Rogie', and 'Scnord Caffing ?'."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2655,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9759,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4752,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 2.6666666666666665,
          "status": "success"
        },
        "flux-2-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/nidsLpCP_dIrgjnMk-zVT_OnzIitB8.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/flux-2-dev/item-003.png",
          "generation_time_ms": 3269,
          "generation_cost_usd": 0.012,
          "scores": {
            "visual_quality": {
              "score": 2.0,
              "confidence": 0.9,
              "explanation": "The image suffers from significant quality issues, primarily the illegible and nonsensical text on the chalkboard menu, which is a common AI artifact. Additionally, a strong light flare on the left side overexposes a large portion of the image, obscuring details and contributing to an overall lack of clarity.",
              "evidence": [
                "The smaller text on the 'COFFEE SHOP' chalkboard is completely unreadable gibberish.",
                "A large, bright light flare on the left side of the image causes significant overexposure and loss of detail.",
                "Many elements, such as the items on the counter and details inside the shop, appear soft and lack sharpness."
              ]
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly adheres to the prompt, featuring a vintage coffee shop storefront with prominent morning sunlight and a clear 35mm film photography style. While a hand-lettered chalkboard menu is present, the text on it is illegible, which is a minor deviation.",
              "evidence": [
                "The image clearly depicts a 'vintage coffee shop storefront' with an old-fashioned aesthetic.",
                "Strong 'morning sunlight' is evident, casting a warm glow and creating lens flare.",
                "The '35mm film photography style' is well-represented through the warm color cast, visible grain, and soft focus.",
                "A 'hand-lettered chalkboard menu' is a central element, though the actual text is nonsensical."
              ]
            },
            "text_rendering": {
              "score": 1.0,
              "confidence": 0.9,
              "explanation": "While the main title 'COFFEE SHOP' is legible, the vast majority of the text on the chalkboard menu consists of garbled, unreadable gibberish. The text does not form coherent words or prices, making the menu content incomprehensible.",
              "evidence": [
                "The title 'COFFEE SHOP' is clearly legible.",
                "Menu items like 'UNTESISTRFOR FRTHTSWIF/B/3', 'BICMAR CR EXTRIRGS', and 'FORIETSHABAK-V83' are completely unreadable and nonsensical."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3557,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8976,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4947,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 2.3333333333333335,
          "status": "success"
        },
        "z-image-turbo": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc19/9z5g_ipeqxPJUkS1omfaZ_erN1Mhsg.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/z-image-turbo/item-003.png",
          "generation_time_ms": 2107,
          "generation_cost_usd": 0.005,
          "scores": {
            "visual_quality": {
              "score": 2.0,
              "confidence": 0.85,
              "explanation": "While the image exhibits good lighting, sharpness, and composition, the primary visual elements\u2014the menu items written on the coffee shop window\u2014are largely incoherent. The text appears as garbled, nonsensical words, significantly detracting from the overall quality and purpose of the displayed information.",
              "evidence": [
                "The menu items in all three window panels (e.g., 'Focus nerg', 'Kanding Gired', 'Bodybiq Secuwgl') are mostly gibberish and not actual readable words.",
                "Despite the clear rendering of the text's font and style, the lack of semantic coherence makes the text elements visually flawed.",
                "The overall image is sharp and well-lit, with realistic reflections, but the central textual content is a major issue."
              ]
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.85,
              "explanation": "The image strongly matches most aspects of the prompt, including the vintage coffee shop storefront, morning sunlight, and 35mm film photography style. The main deviation is the illegible text on the hand-lettered chalkboard menus.",
              "evidence": [
                "The storefront design, dark wood, and overall aesthetic convey a vintage coffee shop feel.",
                "Strong shadows and reflections on the glass clearly indicate morning sunlight.",
                "The image exhibits a slightly desaturated color palette and subtle grain, consistent with a 35mm film photography style.",
                "The menus in the windows are visually hand-lettered on a dark surface, resembling chalkboards, but the text is nonsensical and unreadable."
              ]
            },
            "text_rendering": {
              "score": 2.0,
              "confidence": 0.8,
              "explanation": "While the main headings like 'COFFEE' and 'Morning' are mostly readable and the hand-lettered style is well-rendered, the vast majority of the menu items listed below are garbled and nonsensical words. This makes the core content of the 'chalkboard menu' largely unreadable.",
              "evidence": [
                "Main headings 'COFFEE' (top center), 'Morning' (left panel), and 'Coffee' (right panel) are clear and legible.",
                "The word 'Mornive' in the middle panel is a misspelling of 'Morning'.",
                "Most menu items, such as 'Focus nerg', 'Kanding Gired', 'Bornes Brigess', and 'Bodybiq Secuwgl', are composed of gibberish words, rendering the menu content unreadable.",
                "Only a few isolated words like 'Nuts' and 'eat' are correctly spelled within the extensive list of menu items."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3568,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9745,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4312,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 2.6666666666666665,
          "status": "success"
        },
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc1a/FMW8nPK5wOOXmPW0uygpb_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/ideogram-v3/item-003.png",
          "generation_time_ms": 11662,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 2.0,
              "confidence": 0.9,
              "explanation": "The image suffers from significant text artifacts, with prominent signs displaying nonsensical or misspelled words. While the overall composition and lighting are acceptable, these text issues severely detract from the realism and professional quality of the image.",
              "evidence": [
                "The text on the door reads 'Faces Rertles', which is nonsensical.",
                "The menu board displays 'Palldoff' and 'Coffes', indicating misspellings or garbled text.",
                "The fern appears somewhat artificial, lacking natural leaf texture."
              ]
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly matches the prompt's request for a vintage coffee shop storefront with morning sunlight and a 35mm film photography style. However, the hand-lettered chalkboard menu and door sign contain several noticeable typos and nonsensical words, which detract from the overall quality.",
              "evidence": [
                "The storefront architecture, wooden door, and green trim convey a vintage aesthetic.",
                "Strong, directional shadows on the sidewalk and plant indicate morning sunlight.",
                "The overall color palette and subtle texture suggest a 35mm film photography style.",
                "The chalkboard menu and door sign feature hand-lettered style text, but include misspellings like 'Coffes' and 'Palldoff', and nonsensical phrases like 'Faces Rertles'."
              ]
            },
            "text_rendering": {
              "score": 2.0,
              "confidence": 0.9,
              "explanation": "The text rendering is poor, with significant errors and garbled words. While some words like 'Fresh Brews' and 'PASTRIES' are clear, others are misspelled or completely unreadable.",
              "evidence": [
                "The text on the door window, 'Faces Rertles', is completely garbled and unreadable.",
                "The menu lists 'PALLDOFF' which is a misspelling or garbled word.",
                "The menu lists 'COFFES' which is a misspelling of 'COFFEES'."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2595,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9661,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.5377,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 2.6666666666666665,
          "status": "success"
        },
        "nano-banana": {
          "image_url": "https://v3b.fal.media/files/b/0a91dc1e/JYfbjslpj_fmtYzIsCCEm_nkjB1RpS.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-test-3x5/nano-banana/item-003.png",
          "generation_time_ms": 56429,
          "generation_cost_usd": 0.0398,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.9,
              "explanation": "The image exhibits excellent visual quality with sharp focus, clear details, and realistic rendering of all elements. There are no visible artifacts, distortions, or inconsistencies in lighting or textures.",
              "evidence": [
                "The text on the 'DAILY GRIND MENU' board and 'THE MORNING JOE' window decal is perfectly legible and sharply rendered.",
                "Objects such as the coffee machine, potted plants, and brick sidewalk display realistic textures and coherent forms.",
                "The lighting is natural and consistent, contributing to a professional and high-quality appearance without any noticeable artifacts or blurring."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all aspects of the prompt, depicting a vintage coffee shop storefront with a hand-lettered chalkboard menu, bathed in morning sunlight, and rendered in a convincing 35mm film photography style.",
              "evidence": [
                "The storefront architecture, signage ('THE MORNING JOE'), and overall aesthetic strongly convey a vintage coffee shop feel.",
                "A prominent chalkboard menu with clearly hand-lettered text and prices is displayed.",
                "The warm, golden light and distinct shadows indicate morning sunlight hitting the scene.",
                "The image exhibits characteristics of 35mm film photography, including subtle grain, warm color tones, and a slightly muted, authentic feel."
              ]
            },
            "text_rendering": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "While most of the text, such as 'THE MORNING JOE' and several menu items, is clearly rendered and readable, there is a significant misspelling on the chalkboard menu. The word 'CAPPUCUCE' is incorrectly spelled instead of 'CAPPUCCINO'.",
              "evidence": [
                "The word 'CAPPUCUCE' is misspelled on the chalkboard menu.",
                "The text 'THE MORNING JOE' on the window is clearly rendered and correctly spelled.",
                "Other menu items like 'COFFEE', 'LATTE', and 'PASTRIES' are clearly readable and correctly spelled."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2906,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9329,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "aesthetic_score": {
              "value": 0.4912,
              "description": "Aesthetic quality (LAION MLP on CLIP, 0-1)"
            }
          },
          "overall_score": 4.333333333333333,
          "status": "success"
        }
      }
    }
  ],
  "summary": {
    "flux-schnell": {
      "overall_score": 3.57,
      "dimension_averages": {
        "visual_quality": 3.33,
        "prompt_adherence": 4.67,
        "text_rendering": 3.67
      },
      "dimension_confidence": {
        "visual_quality": 0.87,
        "prompt_adherence": 0.95,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.94,
      "metric_averages": {
        "clip_score": 0.3049,
        "sharpness": 0.9285,
        "aesthetic_score": 0.5033
      },
      "total_generation_cost_usd": 0.009,
      "total_generation_time_ms": 3700,
      "cost_per_image": 0.003,
      "score_per_dollar": 1190.0,
      "dimension_stddev": {
        "visual_quality": 0.94,
        "prompt_adherence": 0.47,
        "text_rendering": 1.89
      },
      "overall_stddev": 1.37,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "aesthetic_score": "included"
      },
      "weighted_overall_score": 3.57
    },
    "flux-2-dev": {
      "overall_score": 4.01,
      "dimension_averages": {
        "visual_quality": 4.0,
        "prompt_adherence": 4.67,
        "text_rendering": 3.67
      },
      "dimension_confidence": {
        "visual_quality": 0.92,
        "prompt_adherence": 0.97,
        "text_rendering": 0.97
      },
      "avg_confidence": 0.95,
      "metric_averages": {
        "clip_score": 0.3509,
        "sharpness": 0.8701,
        "aesthetic_score": 0.5231
      },
      "total_generation_cost_usd": 0.036,
      "total_generation_time_ms": 9873,
      "cost_per_image": 0.012,
      "score_per_dollar": 334.2,
      "dimension_stddev": {
        "visual_quality": 1.41,
        "prompt_adherence": 0.47,
        "text_rendering": 1.89
      },
      "overall_stddev": 1.45,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "aesthetic_score": "included"
      },
      "weighted_overall_score": 4.01
    },
    "z-image-turbo": {
      "overall_score": 4.02,
      "dimension_averages": {
        "visual_quality": 4.0,
        "prompt_adherence": 4.67,
        "text_rendering": 4.0
      },
      "dimension_confidence": {
        "visual_quality": 0.92,
        "prompt_adherence": 0.95,
        "text_rendering": 0.93
      },
      "avg_confidence": 0.93,
      "metric_averages": {
        "clip_score": 0.3464,
        "sharpness": 0.9009,
        "aesthetic_score": 0.5007
      },
      "total_generation_cost_usd": 0.015,
      "total_generation_time_ms": 6795,
      "cost_per_image": 0.005,
      "score_per_dollar": 804.0,
      "dimension_stddev": {
        "visual_quality": 1.41,
        "prompt_adherence": 0.47,
        "text_rendering": 1.41
      },
      "overall_stddev": 1.23,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "aesthetic_score": "included"
      },
      "weighted_overall_score": 4.02
    },
    "ideogram-v3": {
      "overall_score": 3.79,
      "dimension_averages": {
        "visual_quality": 4.0,
        "prompt_adherence": 4.67,
        "text_rendering": 4.0
      },
      "dimension_confidence": {
        "visual_quality": 0.92,
        "prompt_adherence": 0.95,
        "text_rendering": 0.97
      },
      "avg_confidence": 0.94,
      "metric_averages": {
        "clip_score": 0.3,
        "sharpness": 0.9399,
        "aesthetic_score": 0.5288
      },
      "total_generation_cost_usd": 0.09,
      "total_generation_time_ms": 125208,
      "cost_per_image": 0.03,
      "score_per_dollar": 126.3,
      "dimension_stddev": {
        "visual_quality": 1.41,
        "prompt_adherence": 0.47,
        "text_rendering": 1.41
      },
      "overall_stddev": 1.23,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "aesthetic_score": "included"
      },
      "weighted_overall_score": 3.79
    },
    "nano-banana": {
      "overall_score": 4.24,
      "dimension_averages": {
        "visual_quality": 5.0,
        "prompt_adherence": 5.0,
        "text_rendering": 4.33
      },
      "dimension_confidence": {
        "visual_quality": 0.93,
        "prompt_adherence": 1.0,
        "text_rendering": 0.97
      },
      "avg_confidence": 0.97,
      "metric_averages": {
        "clip_score": 0.3137,
        "sharpness": 0.8643,
        "aesthetic_score": 0.5311
      },
      "total_generation_cost_usd": 0.1194,
      "total_generation_time_ms": 117277,
      "cost_per_image": 0.0398,
      "score_per_dollar": 106.5,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.94
      },
      "overall_stddev": 0.63,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "aesthetic_score": "included"
      },
      "weighted_overall_score": 4.24
    }
  },
  "ranking": [
    [
      "nano-banana",
      4.24
    ],
    [
      "z-image-turbo",
      4.02
    ],
    [
      "flux-2-dev",
      4.01
    ],
    [
      "ideogram-v3",
      3.79
    ],
    [
      "flux-schnell",
      3.57
    ]
  ],
  "efficiency_ranking": [
    [
      "flux-schnell",
      1190.0
    ],
    [
      "z-image-turbo",
      804.0
    ],
    [
      "flux-2-dev",
      334.2
    ],
    [
      "ideogram-v3",
      126.3
    ],
    [
      "nano-banana",
      106.5
    ]
  ],
  "winner": "nano-banana",
  "best_value": "flux-schnell",
  "correlation": {
    "clip_score_vs_prompt_adherence": {
      "pearson_r": -0.0784,
      "p_value": 0.957808,
      "interpretation": "low_agreement"
    }
  },
  "cost": {
    "generation": {
      "total_usd": 0.2694,
      "by_model": {
        "flux-schnell": 0.009,
        "flux-2-dev": 0.036,
        "z-image-turbo": 0.015,
        "ideogram-v3": 0.09,
        "nano-banana": 0.1194
      },
      "request_count": 15
    },
    "judge": {
      "total_usd": 0.0225,
      "provider": "gemini-2.5-flash",
      "request_count": 45
    },
    "metrics": {
      "total_usd": 0.0,
      "note": "Local computation"
    },
    "total_usd": 0.2919
  },
  "metadata": {
    "evalytic_version": "0.3.10",
    "python_version": "3.14.3",
    "platform": "darwin-arm64"
  }
}