{
  "$schema": "https://evalytic.ai/schemas/bench-report-v1.json",
  "version": "1.0",
  "name": "bench-7c1c2f89",
  "created_at": "2026-02-27T08:55:15.163101+00:00",
  "duration_seconds": 245.0,
  "config": {
    "models": [
      "flux-schnell",
      "flux-dev",
      "flux-pro"
    ],
    "judge": "gemini-2.5-flash",
    "judges": [
      "gemini-2.5-flash",
      "gemini-3-flash",
      "gemini-2.5-pro"
    ],
    "dimensions": [
      "visual_quality",
      "prompt_adherence",
      "text_rendering"
    ],
    "metrics": [],
    "pipeline": "text2img",
    "seed": null,
    "image_size": null,
    "concurrency": 4
  },
  "items": [
    {
      "item_id": "item-001",
      "prompt": "A golden retriever playing in autumn leaves",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/LXRBf8Hyy0-Wht_DrJtiK.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-schnell/item-001.jpg",
          "generation_time_ms": 1270,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality with sharp focus on the subject, realistic textures for the dog's fur and the autumn leaves, and a pleasing depth of field. There are no visible artifacts or inconsistencies, contributing to a professional and coherent appearance.",
              "evidence": [
                "Sharp focus on the dog's face and front paws, with clear details in the eyes and fur.",
                "Realistic rendering of individual autumn leaves in the foreground, showing varied textures and shapes.",
                "Consistent lighting and natural depth of field, blurring the background appropriately without introducing artifacts.",
                "The dog's body appears slightly compressed or disproportionately short behind the front legs.",
                "The front paws lack clear definition and appear to merge unnaturally with the texture of the autumn leaves.",
                "The metal tag on the collar is blurry and lacks the sharp detail found in the dog's eyes and nose.",
                "The dog's fur texture is highly detailed and realistic, especially on its face and ears.",
                "The lighting is soft and consistent across the scene, and the reflections in the dog's eyes appear natural.",
                "The foreground leaves are sharp and detailed, while the background trees are smoothly blurred, creating a convincing depth of field effect."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 4.0,
                "gemini-2.5-pro": 5.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt, featuring a happy golden retriever in a playful pose surrounded by a thick layer of autumn leaves. The seasonal setting is consistently represented in both the foreground and the blurred background.",
              "evidence": [
                "A golden retriever is clearly depicted with accurate breed characteristics.",
                "The dog is surrounded by a dense layer of autumn leaves, and the background shows trees with fall foliage.",
                "The dog's open-mouthed, happy expression conveys a sense of 'playing' or enjoying the environment.",
                "The subject is clearly a golden retriever with a joyful, open-mouthed expression indicative of play.",
                "The environment is filled with orange and brown autumn leaves, matching the requested setting.",
                "The subject is clearly a golden retriever.",
                "The setting is a forest floor covered in colorful autumn leaves.",
                "The dog is lying down rather than actively playing, which is a slight deviation from the prompt's action verb."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0,
                "gemini-3-flash": 5.0,
                "gemini-2.5-pro": 4.0
              },
              "agreement": "disputed"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected to be rendered in this image based on the provided prompt, which describes the visual content rather than text to be included. The image successfully contains no text.",
              "evidence": [
                "No text is present in the image.",
                "The prompt 'A golden retriever playing in autumn leaves' describes the scene, not text to be rendered.",
                "No text expected.",
                "The image contains only a dog and autumn leaves without any textual elements."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 4.666666666666667,
          "status": "success"
        },
        "flux-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/b7pp1jyOnKayCOoCQs5de.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-dev/item-001.jpg",
          "generation_time_ms": 2593,
          "generation_cost_usd": 0.025,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image exhibits professional quality with no visible artifacts or distortions. The puppy is sharply rendered with realistic fur texture and expressive features, while the background and foreground leaves are beautifully blurred to create a strong sense of depth and focus.",
              "evidence": [
                "Sharp and detailed rendering of the puppy's fur, eyes, and facial features.",
                "Effective use of shallow depth of field, with natural-looking bokeh in the background and foreground leaves.",
                "Consistent and warm lighting that enhances the autumn theme.",
                "The front right paw (viewer's right) features claws that are irregularly shaped and lack realistic anatomical detail.",
                "There is a subtle halo effect or 'cut-out' appearance around the edges of the puppy's ears against the blurred background.",
                "The fur texture on the chest and front legs appears slightly too smooth and lacks the fine detail seen in the facial fur.",
                "The focus is precisely on the puppy's face, with sharp details visible in the eyes, nose, and tongue.",
                "The lighting is natural and soft, creating realistic highlights and shadows on the fur that give it volume and texture.",
                "There are no discernible AI artifacts; the anatomy of the puppy is correct, and the interaction between its paws and the leaves on the ground is convincing."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 4.0,
                "gemini-2.5-pro": 5.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt. It features a golden retriever puppy actively playing amidst a vibrant carpet of autumn leaves, with a warm, autumnal background.",
              "evidence": [
                "The main subject is clearly a golden retriever puppy.",
                "The puppy's posture and expression (tongue out, running) strongly suggest it is 'playing'.",
                "The ground is covered in numerous orange and yellow leaves, accurately representing 'autumn leaves'.",
                "The subject is a golden retriever puppy running towards the camera, conveying a sense of play.",
                "The foreground and ground are covered in detailed, realistic autumn leaves.",
                "The warm color palette and blurred forest background reinforce the autumn theme."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected to be rendered in this image based on the prompt, and indeed, no text is present. The image focuses solely on the visual depiction of a golden retriever in autumn leaves.",
              "evidence": [
                "No visible text elements are present anywhere in the image.",
                "The prompt describes a scene, not text to be generated within the image.",
                "No text expected.",
                "No text artifacts or unintended characters are visible in the image."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 5.0,
          "status": "success"
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/3rZEwuYHQ4KmYzU93Kjdx_6d98ea169b374bc1901de9acee03f685.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-pro/item-001.jpg",
          "generation_time_ms": 6368,
          "generation_cost_usd": 0.05,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image is of excellent, professional quality, closely resembling a high-end photograph. The subject is rendered with exceptional detail and realism, from the texture of the fur to the reflections in the eyes. The composition, lighting, and use of a shallow depth of field are all executed perfectly, resulting in a visually stunning and coherent image with no visible artifacts.",
              "evidence": [
                "The golden retriever is rendered with excellent detail, sharp focus, and natural fur texture.",
                "The depth of field is well-executed, effectively separating the dog from the autumn leaf background.",
                "The foreground leaves, while colorful, appear somewhat generic and lack the intricate variations of real leaves upon close inspection.",
                "Highly detailed fur texture and realistic reflections in the dog's eyes and on its nose.",
                "Consistent and natural-looking depth of field, with sharp foreground leaves transitioning smoothly into a soft background blur.",
                "The focus is perfectly placed on the dog's face, revealing sharp details in the fur, whiskers, and the wet texture of its nose and tongue.",
                "The depth of field is rendered realistically, with a smooth transition from the sharp subject to the softly blurred foreground leaves and background trees, creating a pleasing bokeh effect."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0,
                "gemini-3-flash": 5.0,
                "gemini-2.5-pro": 5.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.95,
              "explanation": "The image perfectly captures the subject and setting, featuring a golden retriever surrounded by a thick layer of autumn leaves. While the dog is lying down rather than engaging in active, dynamic play, its happy expression and the environment strongly align with the prompt's intent.",
              "evidence": [
                "A golden retriever is clearly and accurately depicted.",
                "The dog is surrounded by numerous autumn leaves with appropriate colors.",
                "The dog is lying down, not actively engaged in 'playing' behavior.",
                "The subject is a clearly identifiable golden retriever with accurate breed characteristics.",
                "The environment is filled with vibrant orange and yellow autumn leaves in both the foreground and background.",
                "The dog's pose is static (lying down) rather than showing active movement like running or jumping, which would more literally represent 'playing'."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0,
                "gemini-3-flash": 4.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in this image based on the prompt, and no text is present. The image focuses solely on the visual depiction of a golden retriever in autumn leaves.",
              "evidence": [
                "No text is visible anywhere in the image.",
                "The prompt 'A golden retriever playing in autumn leaves' describes a scene and does not request any text to be rendered.",
                "No text requested in the prompt.",
                "No text visible in the image."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 4.666666666666667,
          "status": "success"
        }
      }
    },
    {
      "item_id": "item-002",
      "prompt": "A futuristic Tokyo street at night with neon signs",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/UlEYnQB_0lENf1RM-0lt3.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-schnell/item-002.jpg",
          "generation_time_ms": 1493,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 1.0,
              "explanation": "The image demonstrates excellent composition, lighting, and atmosphere, creating a convincing nighttime city scene. However, the quality is significantly reduced by the fact that all the text on the numerous signs, which are a central focus of the image, is nonsensical and garbled. While the characters mimic the style of Japanese script, they are not legitimate, which breaks the realism of the scene upon closer inspection.",
              "evidence": [
                "Numerous Japanese characters on various signs are garbled, malformed, or nonsensical (e.g., the large red sign on the right, the blue sign on the far left, and many others).",
                "The people in the scene appear somewhat generic and lack fine detail in their faces and hands, especially those further down the street.",
                "The text on the numerous neon signs is composed of nonsensical, distorted characters that mimic Japanese script but are entirely illegible.",
                "Human figures walking in the street lack facial features and have blurred, indistinct silhouettes.",
                "Close inspection reveals some geometric inconsistencies where signs and architectural elements merge or overlap unnaturally.",
                "The characters on the prominent vertical signs on both the left and right sides of the street are malformed and do not form real words.",
                "The text visible inside the shop on the far left appears to be 'EYGS', which is not a recognizable word."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0,
                "gemini-3-flash": 3.0,
                "gemini-2.5-pro": 3.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly captures the essence of a Tokyo street at night with an abundance of glowing signs. While it perfectly represents the location and lighting, it lacks specific 'futuristic' elements, looking more like a contemporary photograph of modern-day Japan.",
              "evidence": [
                "The street is densely packed with vertical, glowing signs, characteristic of Tokyo's nightlife.",
                "The scene is clearly set at night with artificial illumination from the signs.",
                "There are no discernible futuristic architectural styles, vehicles, or advanced technology present in the image.",
                "The dense vertical signage with Japanese-style characters accurately reflects a Tokyo street scene.",
                "The high-contrast lighting from the numerous backlit signs effectively creates the 'neon signs at night' atmosphere.",
                "The lack of advanced technology, sci-fi architecture, or cyberpunk elements makes the scene feel contemporary rather than futuristic.",
                "The image successfully depicts a street in a dense city with numerous vertical, glowing signs written in Japanese, which is characteristic of Tokyo.",
                "The scene is clearly set at night, with the dark sky contrasting with the bright, colorful lights from the signs and storefronts.",
                "The image lacks overt futuristic elements such as flying cars, holographic advertisements, or advanced architecture, making it appear more modern than futuristic."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.0,
                "gemini-3-flash": 4.0,
                "gemini-2.5-pro": 4.0
              },
              "agreement": "disputed"
            },
            "text_rendering": {
              "score": 1.0,
              "confidence": 1.0,
              "explanation": "The image is filled with text on numerous neon signs, intended to look like Japanese. However, the vast majority of the characters are garbled, malformed, and do not form legible words or phrases. While the AI successfully captures the aesthetic of a neon-lit Japanese street, it fails to render any coherent language.",
              "evidence": [
                "Most Japanese characters on the neon signs are malformed and do not form recognizable words or even correct individual characters.",
                "Examples include the vertical signs on both sides of the street where characters are clearly gibberish, such as the long vertical sign on the far left and many signs in the middle distance.",
                "While some numbers like '2', '10', and '6715' are somewhat readable, the vast majority of the text is not.",
                "The large vertical signs on both sides contain malformed Kanji-like characters that do not correspond to real Japanese text.",
                "Alphanumeric text like 'EYGS' and '6715' is legible but appears as random strings without clear meaning.",
                "Smaller signs in the background dissolve into illegible scribbles and visual artifacts.",
                "Most signs feature text that mimics the style of Japanese characters but is ultimately a jumble of nonsensical strokes and shapes.",
                "On the left, a shop window displays the garbled text 'EYGS'.",
                "On the right, a large vertical red sign contains characters that are completely malformed and do not correspond to any real Japanese script.",
                "While a few numbers like '6715' on a sign on the right are somewhat clear, they are exceptions in an image dominated by unreadable text."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 1.0,
                "gemini-3-flash": 2.0,
                "gemini-2.5-pro": 1.0
              },
              "agreement": "disputed"
            }
          },
          "metrics": {},
          "overall_score": 2.6666666666666665,
          "status": "success"
        },
        "flux-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/MMtWSXO1csfO_kfBQFXMn.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-dev/item-002.jpg",
          "generation_time_ms": 2600,
          "generation_cost_usd": 0.025,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "The image presents a visually appealing night street scene with good lighting and atmospheric reflections. However, the quality is significantly hampered by the pervasive incoherent and unreadable text on almost all prominent neon signs, which is a common AI artifact.",
              "evidence": [
                "Multiple neon signs (e.g., the large pink sign on the left, the tall white sign above it, and several blue signs on the right) display gibberish or malformed characters instead of legible text.",
                "The neon lighting and reflections on the wet street are well-rendered, contributing to a strong atmospheric effect.",
                "The overall composition and depth of the street scene are good, creating a sense of immersion despite the text issues.",
                "The text on all neon signs consists of gibberish characters that mimic East Asian scripts but are illegible and distorted.",
                "The motorcycles and figures parked along the street become increasingly repetitive and lose anatomical/mechanical detail in the mid-ground.",
                "Power lines in the sky are inconsistently rendered, with some lines appearing to float or terminate abruptly without logical connection points."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.0,
                "gemini-3-flash": 3.0
              },
              "agreement": "high"
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image strongly depicts a Tokyo-like street at night, illuminated by numerous vibrant neon signs. While the architecture isn't explicitly futuristic, the intense neon lighting and overall cyberpunk aesthetic successfully convey a futuristic atmosphere.",
              "evidence": [
                "The dense urban street with buildings and numerous vertical signs strongly resembles a Tokyo street.",
                "The dark sky and dominant artificial lighting clearly indicate 'at night'.",
                "Abundant glowing signs in various colors (pink, blue, yellow, white) perfectly match 'neon signs'.",
                "The strong neon reflections on the wet street and overall aesthetic contribute to a 'futuristic' feel, even without overtly futuristic architecture.",
                "Vibrant neon signs in pink and cyan dominate the streetscape.",
                "The narrow street and building architecture are highly characteristic of Tokyo districts like Shinjuku or Shibuya.",
                "The text on the signs is illegible AI-generated gibberish, which is a common artifact but fits the visual aesthetic of a neon-lit city."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0,
                "gemini-3-flash": 4.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 1.0,
              "confidence": 0.95,
              "explanation": "The text on all the neon signs is completely garbled and unreadable. It consists of random strokes and shapes that do not form coherent characters, making it impossible to decipher any meaning.",
              "evidence": [
                "All visible neon signs display garbled, unreadable characters.",
                "The characters are a jumble of strokes, not forming any recognizable language, despite the context of a 'Tokyo street'.",
                "The large vertical signs on both the left and right sides of the street feature pseudo-characters that are linguistically incorrect and visually garbled.",
                "The horizontal signs above the storefronts on the left contain illegible symbols and structural artifacts that do not correspond to any real language."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 1.0,
                "gemini-3-flash": 1.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 2.6666666666666665,
          "status": "success"
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/mbHDM0s1dFqToWRO7VfYE_f939813723b6485598dcc74d4329c2b7.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-pro/item-002.jpg",
          "generation_time_ms": 4956,
          "generation_cost_usd": 0.05,
          "scores": {
            "visual_quality": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "The image effectively captures a vibrant cyberpunk aesthetic with strong lighting and realistic wet-road reflections. However, it suffers from typical AI artifacts such as garbled text on signs and poorly defined background figures, which detract from the overall realism upon closer inspection.",
              "evidence": [
                "Numerous Japanese signs display gibberish or highly distorted characters, particularly visible on the tall red sign on the left and various blue signs on the right.",
                "Human figures in both the foreground and background are blurry and lack distinct features, appearing somewhat generic.",
                "The characters below 'TOKYO' on the main yellow sign are slightly distorted and not perfectly legible.",
                "Nonsensical and distorted characters on the neon signs, particularly visible below the word 'TOKYO'.",
                "Background pedestrians are rendered as blurry, indistinct blobs lacking clear human anatomy.",
                "Inconsistent geometry on the vehicles, such as the distorted taillights and rear bodywork of the silver car in the foreground."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.0,
                "gemini-3-flash": 3.0
              },
              "agreement": "high"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures a futuristic Tokyo street at night, dominated by an abundance of vibrant neon signs. The atmosphere and details align precisely with the prompt's description, creating an excellent visual representation.",
              "evidence": [
                "The street is clearly identifiable as a Tokyo street due to the Japanese script on the numerous signs.",
                "The intense neon lighting and sleek urban environment convey a strong 'futuristic' aesthetic.",
                "The scene is unequivocally set 'at night' with dark skies and illuminated buildings.",
                "The image is saturated 'with neon signs' of various colors and sizes, making them a central feature.",
                "A large prominent sign explicitly displays the word 'TOKYO' alongside Japanese characters.",
                "The street is lined with a high density of glowing neon signs in various colors like pink, cyan, and yellow.",
                "The wet road surface reflects the neon lights, contributing to the classic futuristic/cyberpunk night-time aesthetic."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 3.0,
              "confidence": 0.9,
              "explanation": "The word 'TOKYO' is rendered clearly and correctly on the prominent yellow sign, which is a major focal point. However, the vast majority of the other text in the image consists of garbled, nonsensical characters that merely mimic the appearance of Japanese script.",
              "evidence": [
                "The large yellow sign clearly displays 'TOKYO' in crisp, accurate English text.",
                "Many Japanese characters on various neon signs, such as '\u30d5\u30a3\u30a8\u30c1\u30c4\u30bd' and '\u98a8\u53e3\u30b0\u30e9\u30ec\u30a4' on the red vertical sign, and '\u8d64\u3093\u30b0\u30d5\u30b9' on a blue sign, appear to be plausible-looking but nonsensical combinations of characters.",
                "Some smaller or more distant Japanese text is distorted or incomplete, making it unreadable.",
                "The word 'TOKYO' is clearly legible and correctly spelled on the large yellow oval sign on the left.",
                "The vertical red and blue neon signs contain distorted, pseudo-Japanese characters that do not form real words.",
                "Text on the license plates of the cars and smaller background signs is blurry and illegible."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.0,
                "gemini-3-flash": 3.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 3.6666666666666665,
          "status": "success"
        }
      }
    },
    {
      "item_id": "item-003",
      "prompt": "A steaming cup of coffee on a wooden table, morning light",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/zUs2Ku4jKmuUgyH84eE0F.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-schnell/item-003.jpg",
          "generation_time_ms": 1882,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image exhibits professional quality with excellent lighting, sharp focus on the subject, and realistic textures. There are no visible artifacts or inconsistencies, and the steam from the coffee is rendered naturally.",
              "evidence": [
                "Sharp focus on the coffee cup and the wooden table, with a pleasing depth of field.",
                "Realistic rendering of steam rising from the coffee, appearing natural and ethereal.",
                "Natural and appealing lighting with well-defined shadows and highlights on the wooden surface, enhancing realism.",
                "No visible AI artifacts, distortions, or incoherent elements present in the image.",
                "Highly detailed and realistic wood grain texture on the table surface.",
                "Coherent and aesthetically pleasing rendering of steam rising from the cup.",
                "Consistent lighting and soft-focus background (bokeh) that enhances the subject."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt. There is a steaming cup of coffee, it is placed on a wooden table, and the lighting clearly depicts morning light with warm sunbeams and shadows.",
              "evidence": [
                "A white cup with dark liquid and visible steam rising from it perfectly represents 'A steaming cup of coffee'.",
                "The surface the cup rests on has clear wood grain, matching 'on a wooden table'.",
                "Strong, warm sunlight casting distinct shadows on the table and illuminating the scene accurately portrays 'morning light'.",
                "A white ceramic cup filled with coffee has visible, wispy steam rising from it.",
                "The cup is situated on a rustic wooden table with clear grain and plank textures.",
                "The warm, directional light coming from the side creates long shadows and bright highlights, perfectly evoking a morning atmosphere."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in the image based on the prompt, and indeed, no text is present in the generated image. The image successfully avoids generating any unintended or garbled text.",
              "evidence": [
                "No text is visible anywhere in the image.",
                "The prompt 'A steaming cup of coffee on a wooden table, morning light' does not imply any text should be present.",
                "No text expected.",
                "No text artifacts or garbled characters are visible in the scene."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 5.0,
          "status": "success"
        },
        "flux-dev": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b3/5LxvGjrcgoKjazW50KU4r.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-dev/item-003.jpg",
          "generation_time_ms": 2092,
          "generation_cost_usd": 0.025,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality with professional-grade rendering. The lighting is beautifully executed, creating a warm and inviting atmosphere, and the steam rising from the cup appears highly realistic.",
              "evidence": [
                "The steam is rendered with naturalistic wisps and translucency, effectively conveying heat.",
                "The backlighting creates a strong sense of depth and highlights the cup and steam beautifully.",
                "The bokeh in the background is smooth and natural, drawing focus to the main subject without distractions.",
                "The cup and saucer are well-defined with appropriate reflections and shadows, indicating clean rendering.",
                "The rim and edge of the saucer are wobbly and lack a consistent circular curvature.",
                "There is an incoherent white artifact/highlight on the lower left side of the cup's body that does not align with the light source or the cup's geometry.",
                "The connection point where the handle meets the cup body is slightly distorted and lacks a clean transition.",
                "The backlighting creates a realistic rim light on the top edge of the cup and handle.",
                "The steam is rendered with natural, wispy details and is well-integrated into the scene's lighting.",
                "The shallow depth of field is executed flawlessly, with a sharp subject and a smoothly blurred background, enhancing the composition."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 3.0,
                "gemini-2.5-pro": 5.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image is a perfect visual representation of the prompt, capturing every requested element with high fidelity. The steam, the wooden texture of the table, and the specific quality of morning light are all clearly present and well-executed.",
              "evidence": [
                "A visible stream of vapor rises from the cup, indicating 'steaming coffee'.",
                "The cup is placed on a surface with clear wood grain, matching 'wooden table'.",
                "The warm, directional light with strong highlights and shadows is characteristic of 'morning light'.",
                "Prominent steam rising from the coffee cup",
                "The cup is resting on a clearly defined wooden table surface",
                "Warm, golden-hour lighting and long shadows accurately depict morning light"
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected to be rendered in this image based on the provided prompt. The image successfully depicts the scene without any text elements.",
              "evidence": [
                "No text is present in the image.",
                "The prompt describes a scene ('A steaming cup of coffee on a wooden table, morning light') and does not request any text to be generated within the image.",
                "No text expected based on the prompt.",
                "No text elements are visible in the image."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 5.0,
          "status": "success"
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a9022b4/fRfAg21hsYzrkqN4S1T3h_21738d75290a4c39817adacd44358457.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/bench-7c1c2f89/flux-pro/item-003.jpg",
          "generation_time_ms": 5092,
          "generation_cost_usd": 0.05,
          "scores": {
            "visual_quality": {
              "score": 4.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality, characterized by professional rendering, sharp focus on the main subject, and a highly coherent composition. The lighting and atmospheric effects are particularly well-executed, contributing to a realistic and appealing scene.",
              "evidence": [
                "The steam rising from the coffee is rendered with realistic detail and movement.",
                "The golden hour lighting creates a beautiful, natural glow and realistic shadows on the wooden table.",
                "The background features a natural-looking bokeh effect, effectively separating the subject and adding depth.",
                "The textures of the wooden table, coffee cup, and coffee beans are detailed and consistent.",
                "The steam rising from the cup has a highly stylized, ornamental swirl pattern that appears more like a graphic element than natural vapor.",
                "The coffee beans on the wooden table exhibit inconsistent shapes and smooth textures that lack the characteristic cracks and oily sheen of real roasted beans.",
                "The handle of the cup has a slightly soft, blended connection point to the main body, typical of AI-generated ceramics.",
                "The steam rising from the coffee is rendered in an overly ornate and perfectly uniform pattern, which appears unnatural compared to real steam.",
                "The warm, directional sunlight is rendered beautifully, creating realistic highlights on the ceramic cup and long, soft shadows on the textured wooden table."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 4.0,
                "gemini-2.5-pro": 4.0
              },
              "agreement": "disputed"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures all elements of the prompt. There is a steaming cup of coffee, placed on a wooden table, illuminated by warm morning light.",
              "evidence": [
                "A white cup with visible steam rising from it is prominently featured.",
                "The cup is resting on a surface clearly identifiable as a wooden table with visible grain.",
                "The lighting is warm, golden, and casts long shadows, indicative of morning light.",
                "A white ceramic cup with prominent, artistic steam rising from the liquid.",
                "The cup is placed on a detailed wooden table with visible grain and texture.",
                "Warm, golden directional light from the background creates a convincing morning glow."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected or generated in this image. The prompt described a visual scene rather than specific text to be rendered.",
              "evidence": [
                "No visible text elements are present in the image.",
                "The prompt 'A steaming cup of coffee on a wooden table, morning light' describes a scene, not text content.",
                "No text expected.",
                "No garbled or unintended text artifacts are visible in the scene."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0,
                "gemini-3-flash": 5.0
              },
              "agreement": "high"
            }
          },
          "metrics": {},
          "overall_score": 4.666666666666667,
          "status": "success"
        }
      }
    }
  ],
  "summary": {
    "flux-schnell": {
      "overall_score": 4.11,
      "dimension_averages": {
        "visual_quality": 4.33,
        "prompt_adherence": 4.33,
        "text_rendering": 3.67
      },
      "dimension_confidence": {
        "visual_quality": 0.98,
        "prompt_adherence": 0.97,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.98,
      "metric_averages": {},
      "total_generation_cost_usd": 0.009,
      "total_generation_time_ms": 4645,
      "cost_per_image": 0.003,
      "score_per_dollar": 1370.0,
      "dimension_stddev": {
        "visual_quality": 0.94,
        "prompt_adherence": 0.47,
        "text_rendering": 1.89
      },
      "overall_stddev": 1.29,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "agreement_rate": 0.44
    },
    "flux-dev": {
      "overall_score": 4.22,
      "dimension_averages": {
        "visual_quality": 4.33,
        "prompt_adherence": 4.67,
        "text_rendering": 3.67
      },
      "dimension_confidence": {
        "visual_quality": 0.95,
        "prompt_adherence": 0.97,
        "text_rendering": 0.98
      },
      "avg_confidence": 0.97,
      "metric_averages": {},
      "total_generation_cost_usd": 0.075,
      "total_generation_time_ms": 7285,
      "cost_per_image": 0.025,
      "score_per_dollar": 168.8,
      "dimension_stddev": {
        "visual_quality": 0.94,
        "prompt_adherence": 0.47,
        "text_rendering": 1.89
      },
      "overall_stddev": 1.31,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "agreement_rate": 0.78
    },
    "flux-pro": {
      "overall_score": 4.33,
      "dimension_averages": {
        "visual_quality": 4.0,
        "prompt_adherence": 4.67,
        "text_rendering": 4.33
      },
      "dimension_confidence": {
        "visual_quality": 0.95,
        "prompt_adherence": 0.98,
        "text_rendering": 0.97
      },
      "avg_confidence": 0.97,
      "metric_averages": {},
      "total_generation_cost_usd": 0.15,
      "total_generation_time_ms": 16416,
      "cost_per_image": 0.05,
      "score_per_dollar": 86.6,
      "dimension_stddev": {
        "visual_quality": 0.82,
        "prompt_adherence": 0.47,
        "text_rendering": 0.94
      },
      "overall_stddev": 0.82,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "agreement_rate": 0.78
    }
  },
  "ranking": [
    [
      "flux-pro",
      4.33
    ],
    [
      "flux-dev",
      4.22
    ],
    [
      "flux-schnell",
      4.11
    ]
  ],
  "efficiency_ranking": [
    [
      "flux-schnell",
      1370.0
    ],
    [
      "flux-dev",
      168.8
    ],
    [
      "flux-pro",
      86.6
    ]
  ],
  "winner": "flux-pro",
  "best_value": "flux-schnell",
  "correlation": {},
  "cost": {
    "generation": {
      "total_usd": 0.234,
      "by_model": {
        "flux-schnell": 0.009,
        "flux-dev": 0.075,
        "flux-pro": 0.15
      },
      "request_count": 9
    },
    "judge": {
      "total_usd": 0.072,
      "provider": "consensus:gemini-2.5-flash,gemini-3-flash,gemini-2.5-pro",
      "request_count": 63,
      "providers": [
        "gemini-2.5-flash",
        "gemini-3-flash",
        "gemini-2.5-pro"
      ],
      "cost_by_provider": {
        "gemini-2.5-flash": 0.0135,
        "gemini-3-flash": 0.0135,
        "gemini-2.5-pro": 0.045
      }
    },
    "metrics": {
      "total_usd": 0.0,
      "note": "Local computation"
    },
    "total_usd": 0.306
  },
  "metadata": {
    "evalytic_version": "0.2.1",
    "python_version": "3.14.3",
    "platform": "darwin-arm64"
  },
  "judges": [
    "gemini-2.5-flash",
    "gemini-3-flash",
    "gemini-2.5-pro"
  ],
  "consensus_mode": true
}