{
  "$schema": "https://evalytic.ai/schemas/bench-report-v1.json",
  "version": "1.0",
  "name": "nima-ocr-test-3x3",
  "created_at": "2026-03-13T12:43:38.184690+00:00",
  "duration_seconds": 78.9,
  "config": {
    "models": [
      "ideogram-v3",
      "recraft-v3",
      "flux-pro"
    ],
    "judge": "gemini-2.5-flash",
    "judges": [
      "gemini-2.5-flash",
      "gpt-5.2"
    ],
    "dimensions": [
      "visual_quality",
      "prompt_adherence",
      "text_rendering"
    ],
    "metrics": [
      "clip",
      "sharpness",
      "nima",
      "ocr"
    ],
    "pipeline": "text2img",
    "seed": 42,
    "image_size": null,
    "concurrency": 6,
    "scoring_config": {
      "clip_score": {
        "flag_threshold": 0.18,
        "weight": 0.2,
        "normalize_range": [
          0.18,
          0.35
        ]
      },
      "lpips": {
        "flag_threshold": 0.4,
        "weight": 0.2,
        "normalize_range": [
          0.4,
          0.95
        ]
      },
      "face_similarity": {
        "flag_threshold": 0.6,
        "weight": 0.2,
        "normalize_range": [
          0.6,
          0.95
        ]
      },
      "nima_score": {
        "flag_threshold": 0.3,
        "weight": 0.15,
        "normalize_range": [
          0.3,
          0.8
        ]
      },
      "ocr_accuracy": {
        "flag_threshold": 0.0,
        "weight": 0.1,
        "normalize_range": [
          0.0,
          1.0
        ]
      }
    }
  },
  "items": [
    {
      "item_id": "test-port",
      "prompt": "A bearded blacksmith in a dark workshop, sparks flying from the anvil, cinematic side lighting",
      "results": {
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/5zranBQ7VmuLPdz5c60Ix_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/ideogram-v3/test-port.png",
          "generation_time_ms": 11806,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 4.8,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality with sharp details, realistic lighting, and dynamic effects like sparks and hammer motion blur. The rendering of the blacksmith, anvil, and hot metal is highly convincing, contributing to a professional-grade appearance.",
              "evidence": [
                "Sharp rendering of the blacksmith's beard, skin, and anvil details.",
                "Realistic warm glow from the hot metal reflecting on the arm and apron.",
                "Convincing sparks and appropriate motion blur on the hammer.",
                "Minor slight lack of definition in some background textures (e.g., brick wall on the left, coal pile on the right)."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.8
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.9,
              "confidence": 0.95,
              "explanation": "The image is an excellent match for the prompt, depicting a bearded blacksmith in a dark workshop with sparks flying from the anvil. The cinematic side lighting is effectively portrayed through the dramatic light beam from the window and the strong glow from the hot metal, creating a high-contrast, atmospheric scene.",
              "evidence": [
                "A clearly bearded blacksmith is the central subject, wearing an apron and working.",
                "The setting is a dimly lit, rustic workshop with brick walls and tools.",
                "Numerous bright sparks are visibly flying from the point of impact on the anvil.",
                "A strong, dramatic beam of light enters from the right side, illuminating dust and creating high contrast, fulfilling the 'cinematic side lighting' request."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.9
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The prompt did not request any text to be present in the image. As such, the absence of text is perfectly aligned with the prompt's requirements.",
              "evidence": [
                "No text is present in the image.",
                "The prompt 'A bearded blacksmith in a dark workshop, sparks flying from the anvil, cinematic side lighting' does not specify any text content."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.308,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9061,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.524,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 4.8999999999999995,
          "status": "success"
        },
        "recraft-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009e/FgnM2pPF1eDOsPz-vSg0z_image.webp",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/recraft-v3/test-port.webp",
          "generation_time_ms": 5599,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 4.7,
              "confidence": 0.9,
              "explanation": "The image exhibits excellent visual quality with dramatic lighting, realistic sparks, and a well-rendered subject. The composition and atmosphere are highly effective, creating a compelling scene.",
              "evidence": [
                "The sparks are dynamic and realistic, showing appropriate glow and light interaction with the surrounding elements.",
                "The blacksmith's face, beard, and hands are detailed and naturally rendered, conveying concentration.",
                "The overall lighting is atmospheric and enhances the scene without obscuring important details, contributing to a professional aesthetic."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.7
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.8,
              "confidence": 0.95,
              "explanation": "The image perfectly captures all elements of the prompt: a bearded blacksmith, a dark workshop, and sparks flying from the anvil. The lighting is indeed cinematic and comes from the side, primarily from the sparks themselves, creating a dramatic effect.",
              "evidence": [
                "The subject is clearly a bearded man, consistent with a blacksmith, wearing an apron and gloves.",
                "The setting is a dimly lit, dark workshop with a window in the background.",
                "Numerous bright sparks are actively flying from the anvil where the blacksmith is working.",
                "The primary illumination on the blacksmith's face and the immediate workspace is dramatic side lighting originating from the sparks."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.8
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in the image based on the prompt, and no text is present in the generated image. Therefore, the text rendering quality is perfect.",
              "evidence": [
                "No text is visible anywhere in the image.",
                "The prompt did not specify any text to be included."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.363,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.945,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.546,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 4.833333333333333,
          "status": "success"
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/2J2G_F6dIXVyIjmQeENKC_a282ef5b691b461893419217366a6240.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/flux-pro/test-port.jpg",
          "generation_time_ms": 4044,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 4.8,
              "confidence": 0.95,
              "explanation": "The image exhibits professional-level quality with excellent dramatic lighting, sharp details, and realistic rendering of the subject and sparks. There are no visible artifacts or inconsistencies, contributing to a highly coherent and visually appealing scene.",
              "evidence": [
                "Sharp focus on the blacksmith, anvil, and dynamic sparks.",
                "Realistic rendering of light and glow from the hot metal illuminating the subject's face and arms.",
                "Absence of visible artifacts, blurring, or anatomical distortions in the main subject."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.8
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.9,
              "confidence": 0.95,
              "explanation": "The image perfectly captures all elements of the prompt: a bearded blacksmith, a dark workshop, sparks flying from the anvil, and cinematic side lighting. The execution is highly detailed and visually striking, leaving very little room for improvement in matching the prompt's description.",
              "evidence": [
                "A bearded blacksmith is prominently featured, with a well-defined beard and appropriate attire.",
                "The setting is clearly a dark workshop, with ambient light only highlighting the subject and action.",
                "Numerous bright sparks are actively flying from the anvil, indicating intense work.",
                "The lighting is dramatic and high-contrast, illuminating the subject from the side, consistent with 'cinematic side lighting'."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.9
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "No text was expected in this image based on the prompt, and no text is present. The image successfully depicts the visual scene described without any text elements.",
              "evidence": [
                "No discernible text elements are present in the image.",
                "The prompt describes a visual scene ('A bearded blacksmith in a dark workshop, sparks flying from the anvil, cinematic side lighting') and does not request any text to be rendered."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.34,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.8879,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.5593,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 4.8999999999999995,
          "status": "success"
        }
      }
    },
    {
      "item_id": "test-scene",
      "prompt": "A vintage coffee shop storefront with hand-lettered chalkboard menu, morning sunlight, 35mm film photography style",
      "results": {
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/-3O4EHjMIrZSMrxbWsOg7_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/ideogram-v3/test-scene.png",
          "generation_time_ms": 11261,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image demonstrates high visual quality with excellent lighting, sharp details, and realistic rendering of most elements like the bicycle and plants. However, the presence of incoherent, gibberish text on the lower part of the chalkboard sign and other smaller signs on the door detracts from an otherwise strong visual presentation.",
              "evidence": [
                "Sharp rendering of the bicycle and potted ferns, contributing to overall realism.",
                "Legible and well-styled 'Fresh BREWS Daily' text on the chalkboard.",
                "Incoherent and gibberish text 'SUADJEC GUE TVOIS COFFE COFFEE' on the chalkboard, which is a clear artifact.",
                "Unreadable and distorted text on smaller signs attached to the door and visible through the window."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.8,
              "confidence": 0.95,
              "explanation": "The image perfectly captures a vintage coffee shop storefront with a clear hand-lettered chalkboard menu. The lighting strongly suggests morning sunlight with distinct shadows, and the overall aesthetic successfully conveys a 35mm film photography style.",
              "evidence": [
                "The storefront features distressed paint and classic architecture, aligning with a 'vintage' feel.",
                "A prominent chalkboard displays 'Fresh BREWS Daily' in a clear hand-lettered style.",
                "Long, distinct shadows on the sidewalk and warm lighting indicate 'morning sunlight'.",
                "The image exhibits a slightly desaturated color palette and subtle grain, consistent with a '35mm film photography style'."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.8
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 2.5,
              "confidence": 0.8,
              "explanation": "While the prominent 'Fresh BREWS Daily' text on the chalkboard is well-rendered and legible, other text elements in the image are significantly flawed. The smaller text below the main phrase on the chalkboard is garbled and nonsensical, and the text on the signs affixed to the shop door is completely unreadable.",
              "evidence": [
                "The main text 'Fresh BREWS Daily' on the chalkboard is clear and correctly rendered.",
                "The smaller text 'SUADJEC GUE TVOIS COFFE COFFEE' on the chalkboard is garbled and contains misspellings.",
                "Text on the door signs (e.g., 'AART HURES WORKS' and other smaller labels) is unreadable and appears as random characters or blurry shapes."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 2.5
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2305,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9595,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.4941,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 3.766666666666667,
          "status": "success"
        },
        "recraft-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009e/uWoCWB06pr3sW1v6kSyuL_image.webp",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/recraft-v3/test-scene.webp",
          "generation_time_ms": 6771,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image exhibits professional quality with no visible artifacts, distortions, or inconsistencies. The details are sharp, lighting is natural, and textures are realistic, making it indistinguishable from a high-quality photograph.",
              "evidence": [
                "The text on the 'Coffee Shop' sign is perfectly clear and well-formed.",
                "Textures of the wood, brick, and building siding are highly realistic and detailed.",
                "Lighting is natural and evenly distributed, creating a pleasant and authentic scene."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image successfully captures a vintage coffee shop storefront with warm, natural lighting and a film photography aesthetic. However, it deviates from the prompt by featuring a hand-lettered wooden sign instead of a hand-lettered chalkboard menu.",
              "evidence": [
                "The image clearly depicts a 'vintage coffee shop storefront' with classic architecture and wooden details.",
                "The lighting suggests 'morning sunlight' with bright areas and soft shadows.",
                "The overall aesthetic, including color tones and depth, aligns with a '35mm film photography style'.",
                "The prompt requested a 'hand-lettered chalkboard menu', but the image shows a 'hand-lettered wooden sign' and no visible menu."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.0
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'Coffee Shop' on the storefront sign is perfectly rendered, crisp, and accurate. The hand-lettered style is well-executed with no visible artifacts or errors.",
              "evidence": [
                "The words 'Coffee Shop' are clearly legible and sharp.",
                "The font style is consistent and free of rendering imperfections."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2576,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9732,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.4641,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 4.666666666666667,
          "status": "success"
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/kqp-h_6IlRuy5PIRq3cTp_4feba12a24a140efbec76bad9af8db47.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/flux-pro/test-scene.jpg",
          "generation_time_ms": 3648,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 3.7,
              "confidence": 0.9,
              "explanation": "The image presents a visually appealing coffee shop scene with good lighting and composition. However, the quality is significantly impacted by inconsistent and often illegible text on multiple signs, which is a common AI artifact.",
              "evidence": [
                "Text on the chalkboard menu is mostly illegible and malformed, with jumbled letters and numbers.",
                "The small wooden sign on the left has blurry and indistinct text ('Canna Cova Shed').",
                "The sign visible inside the shop through the window also contains malformed text ('Cana Coffee').",
                "Overall good lighting and realistic depiction of the shop's exterior and interior elements."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.7
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.6,
              "confidence": 0.9,
              "explanation": "The image strongly adheres to all aspects of the prompt, depicting a vintage coffee shop storefront with a hand-lettered chalkboard menu and clear morning sunlight. The overall aesthetic also successfully conveys a 35mm film photography style.",
              "evidence": [
                "The storefront design, awning, and wooden elements create a convincing vintage coffee shop aesthetic.",
                "A prominent chalkboard menu is present, featuring text that appears to be hand-lettered, although some words are slightly garbled.",
                "Strong, directional shadows on the ground and lower facade clearly indicate morning sunlight.",
                "The image exhibits a warm color palette, natural depth of field, and a slightly soft quality consistent with 35mm film photography."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.6
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 2.2,
              "confidence": 0.9,
              "explanation": "While the main 'Coffee' and 'Coffee Shop' signs are clearly rendered and readable, the prominent chalkboard menu, a key text element mentioned in the prompt, is largely unreadable gibberish. Other smaller text elements, such as the wooden sign and text visible inside the shop, are also garbled.",
              "evidence": [
                "The chalkboard menu items are mostly unreadable gibberish (e.g., 'Chuk Chra', 'Chaffec', 'Creeee'), failing to convey coherent information.",
                "A small wooden sign on the left side of the storefront displays nonsensical text like 'Canna Cova Shul'.",
                "Text visible inside the shop through the window, such as 'Cana Coffea', is also gibberish.",
                "The main 'Coffee' sign above the door and 'Coffee Shop' sign above the window are clearly rendered and correct."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 2.2
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2316,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9709,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.5088,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            }
          },
          "overall_score": 3.5,
          "status": "success"
        }
      }
    },
    {
      "item_id": "test-text",
      "prompt": "A glowing neon sign reading OPEN 24 HOURS in a dark rainy alley, reflections on wet cobblestones",
      "results": {
        "ideogram-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/fAaSIsXPOqZiefU3jMEgw_image.png",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/ideogram-v3/test-text.png",
          "generation_time_ms": 12939,
          "generation_cost_usd": 0.03,
          "scores": {
            "visual_quality": {
              "score": 4.7,
              "confidence": 0.9,
              "explanation": "The image exhibits excellent visual quality with sharp details, realistic lighting, and convincing reflections on the wet pavement. The neon sign is rendered perfectly, and the overall atmosphere is coherent and immersive.",
              "evidence": [
                "The 'OPEN 24 HOURS' neon sign is perfectly legible and sharply rendered with a realistic glow.",
                "The reflections of the neon light on the wet cobblestone street are highly realistic and consistent with the light source.",
                "No significant artifacts, distortions, or incoherent elements are present throughout the image, even in the blurred background."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.7
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.5,
              "confidence": 0.9,
              "explanation": "The image strongly adheres to the prompt, featuring a perfectly rendered glowing neon sign reading 'OPEN 24 HOURS' and excellent reflections on wet cobblestones. The scene is dark and clearly depicts a rainy environment. The only minor deviation is that the setting feels more like a narrow, wet street than a typical 'alley'.",
              "evidence": [
                "The neon sign clearly reads 'OPEN 24 HOURS' and is glowing brightly.",
                "There are prominent and realistic reflections of the neon light and other ambient lights on the wet cobblestone street.",
                "The environment is dark and appears wet, consistent with a rainy setting.",
                "The location is a narrow street, which is a close interpretation of an 'alley' but perhaps not as secluded or unkempt as some might envision an alley."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.5
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text on the neon sign, 'OPEN 24 HOURS', is perfectly rendered, crisp, and accurate, matching the expected content. Additionally, the watermark text 'Your city. Your fuel.' and the background text 'WHITE OTTO' are also perfectly legible and well-formed.",
              "evidence": [
                "The neon sign clearly displays 'OPEN 24 HOURS' with no distortions or artifacts.",
                "The watermark 'Your city. Your fuel.' at the bottom is sharp and perfectly readable.",
                "Faint text 'WHITE OTTO' on a building in the background is also legible."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2464,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9364,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.5611,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            },
            "ocr_accuracy": {
              "value": 0.0,
              "description": "OCR text accuracy (SequenceMatcher ratio, 0-1)"
            }
          },
          "overall_score": 4.733333333333333,
          "status": "success",
          "flags": [
            {
              "metric": "ocr_accuracy",
              "value": 0.0,
              "threshold": 0.0,
              "message": "OCR accuracy (0.0000) below threshold (0.0) \u2014 excluded from overall"
            }
          ]
        },
        "recraft-v3": {
          "image_url": "https://v3b.fal.media/files/b/0a92009f/RI7bGFiPc9ioL4Cvx_yDD_image.webp",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/recraft-v3/test-text.webp",
          "generation_time_ms": 10222,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 4.7,
              "confidence": 0.9,
              "explanation": "The image demonstrates excellent visual quality, particularly in its realistic rendering of neon lights and their reflections on the wet pavement. The atmospheric night scene with subtle rain is well-executed, and there are no noticeable artifacts or inconsistencies.",
              "evidence": [
                "Realistic glow and reflections of the neon 'OPEN 24 HOURS' sign on the wet cobblestone street.",
                "Consistent and convincing depiction of rain and wet surfaces, enhancing the atmospheric quality.",
                "Sharp focus on the main subject and foreground, with appropriate depth of field blurring in the background."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.7
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 4.6,
              "confidence": 0.9,
              "explanation": "The image strongly adheres to the prompt, featuring a clear, glowing 'OPEN 24 HOURS' neon sign with excellent reflections on the wet, cobblestone-like ground. The dark and rainy atmosphere is perfectly captured, though the setting appears more like a city sidewalk or street rather than a narrow alley.",
              "evidence": [
                "The image clearly shows a glowing neon sign reading 'OPEN 24 HOURS'.",
                "The scene is dark and visibly rainy, with rain streaks and wet surfaces.",
                "There are prominent reflections of the neon sign and distant lights on the wet, cobblestone-patterned ground.",
                "The setting is an urban street/sidewalk, which is a slight deviation from the specified 'dark rainy alley'."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.6
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'OPEN 24 HOURS' is perfectly rendered, crisp, and accurate. Each letter and number is clearly legible with no distortions or artifacts, and the neon effect is well-executed.",
              "evidence": [
                "The word 'OPEN' is correctly spelled and clearly visible.",
                "The numbers '24' are perfectly formed and distinct.",
                "The word 'HOURS' is correctly spelled and highly legible.",
                "The neon glow effect is consistent and enhances readability without blurring the characters."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.2823,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9376,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.538,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            },
            "ocr_accuracy": {
              "value": 0.0,
              "description": "OCR text accuracy (SequenceMatcher ratio, 0-1)"
            }
          },
          "overall_score": 4.766666666666667,
          "status": "success",
          "flags": [
            {
              "metric": "ocr_accuracy",
              "value": 0.0,
              "threshold": 0.0,
              "message": "OCR accuracy (0.0000) below threshold (0.0) \u2014 excluded from overall"
            }
          ]
        },
        "flux-pro": {
          "image_url": "https://v3b.fal.media/files/b/0a9200a0/NB3512p1nhePSyKZNOl8I_25e77dc12cd7434fb2370d12c7c94c07.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/nima-ocr-test-3x3/flux-pro/test-text.jpg",
          "generation_time_ms": 9265,
          "generation_cost_usd": 0.04,
          "scores": {
            "visual_quality": {
              "score": 3.9,
              "confidence": 0.85,
              "explanation": "The image boasts excellent atmospheric lighting and realistic reflections on the wet ground, creating a strong and immersive mood. However, its visual quality is significantly impacted by prominent text artifacts, including a clear misspelling and garbled, illegible text on several signs.",
              "evidence": [
                "The sign on the left side of the street clearly displays the misspelling 'Opeen' instead of 'Open'.",
                "Text on other signs, such as the one in the upper right corner, is garbled and illegible (e.g., 'D8 P 5F4 W590').",
                "The main 'OPEN 24 HOURS' neon sign is sharp and perfectly legible.",
                "Realistic and vibrant reflections of neon lights are accurately rendered on the wet cobblestone street."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 3.9
              },
              "agreement": "degraded"
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly matches all aspects of the prompt. The glowing neon sign reading 'OPEN 24 HOURS' is prominently featured in a dark, rainy alley, with excellent reflections on the wet cobblestones.",
              "evidence": [
                "The main sign clearly reads 'OPEN 24 HOURS' and is glowing neon.",
                "The setting is a dark alleyway between buildings, consistent with a 'dark rainy alley'.",
                "Vivid red and white reflections are clearly visible on the wet, cobblestone-like ground, fulfilling 'reflections on wet cobblestones'."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 5.0
              },
              "agreement": "degraded"
            },
            "text_rendering": {
              "score": 4.2,
              "confidence": 0.9,
              "explanation": "The primary neon sign 'OPEN 24 HOURS' is perfectly rendered, crisp, and accurate, matching the expected text content. However, other text elements within the image are poorly rendered, including a misspelled 'Opeen' sign and several other signs with garbled or unreadable text.",
              "evidence": [
                "The main neon sign clearly and correctly displays 'OPEN 24 HOURS'.",
                "A smaller sign to the left of the main sign reads 'Opeen', which is a misspelling of 'Open'.",
                "Multiple other signs in the background and reflections contain unreadable or garbled text."
              ],
              "judge_scores": {
                "gemini-2.5-flash": 4.2
              },
              "agreement": "degraded"
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3231,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9296,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            },
            "nima_score": {
              "value": 0.5492,
              "description": "NIMA aesthetic quality (AVA-trained, 0-1)"
            },
            "ocr_accuracy": {
              "value": 0.0,
              "description": "OCR text accuracy (SequenceMatcher ratio, 0-1)"
            }
          },
          "overall_score": 4.366666666666666,
          "status": "success",
          "flags": [
            {
              "metric": "ocr_accuracy",
              "value": 0.0,
              "threshold": 0.0,
              "message": "OCR accuracy (0.0000) below threshold (0.0) \u2014 excluded from overall"
            }
          ]
        }
      }
    }
  ],
  "summary": {
    "ideogram-v3": {
      "overall_score": 3.73,
      "dimension_averages": {
        "visual_quality": 4.5,
        "prompt_adherence": 4.73,
        "text_rendering": 4.17
      },
      "dimension_confidence": {
        "visual_quality": 0.92,
        "prompt_adherence": 0.93,
        "text_rendering": 0.93
      },
      "avg_confidence": 0.93,
      "metric_averages": {
        "clip_score": 0.2616,
        "sharpness": 0.934,
        "nima_score": 0.5264,
        "ocr_accuracy": 0.0
      },
      "total_generation_cost_usd": 0.09,
      "total_generation_time_ms": 36006,
      "cost_per_image": 0.03,
      "score_per_dollar": 124.3,
      "dimension_stddev": {
        "visual_quality": 0.36,
        "prompt_adherence": 0.17,
        "text_rendering": 1.18
      },
      "overall_stddev": 0.75,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "nima_score": "included",
        "ocr_accuracy": "flagged"
      },
      "weighted_overall_score": 3.73,
      "agreement_rate": 0.0
    },
    "recraft-v3": {
      "overall_score": 4.13,
      "dimension_averages": {
        "visual_quality": 4.8,
        "prompt_adherence": 4.47,
        "text_rendering": 5.0
      },
      "dimension_confidence": {
        "visual_quality": 0.93,
        "prompt_adherence": 0.92,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.95,
      "metric_averages": {
        "clip_score": 0.301,
        "sharpness": 0.9519,
        "nima_score": 0.516,
        "ocr_accuracy": 0.0
      },
      "total_generation_cost_usd": 0.12,
      "total_generation_time_ms": 22592,
      "cost_per_image": 0.04,
      "score_per_dollar": 103.2,
      "dimension_stddev": {
        "visual_quality": 0.14,
        "prompt_adherence": 0.34,
        "text_rendering": 0.0
      },
      "overall_stddev": 0.31,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "nima_score": "included",
        "ocr_accuracy": "flagged"
      },
      "weighted_overall_score": 4.13,
      "agreement_rate": 0.0
    },
    "flux-pro": {
      "overall_score": 3.82,
      "dimension_averages": {
        "visual_quality": 4.13,
        "prompt_adherence": 4.83,
        "text_rendering": 3.8
      },
      "dimension_confidence": {
        "visual_quality": 0.9,
        "prompt_adherence": 0.95,
        "text_rendering": 0.93
      },
      "avg_confidence": 0.93,
      "metric_averages": {
        "clip_score": 0.2982,
        "sharpness": 0.9295,
        "nima_score": 0.5391,
        "ocr_accuracy": 0.0
      },
      "total_generation_cost_usd": 0.12,
      "total_generation_time_ms": 16957,
      "cost_per_image": 0.04,
      "score_per_dollar": 95.5,
      "dimension_stddev": {
        "visual_quality": 0.48,
        "prompt_adherence": 0.17,
        "text_rendering": 1.18
      },
      "overall_stddev": 0.86,
      "sample_count": 3,
      "total_items": 3,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included",
        "nima_score": "included",
        "ocr_accuracy": "flagged"
      },
      "weighted_overall_score": 3.82,
      "agreement_rate": 0.0
    }
  },
  "ranking": [
    [
      "recraft-v3",
      4.13
    ],
    [
      "flux-pro",
      3.82
    ],
    [
      "ideogram-v3",
      3.73
    ]
  ],
  "efficiency_ranking": [
    [
      "ideogram-v3",
      124.3
    ],
    [
      "recraft-v3",
      103.2
    ],
    [
      "flux-pro",
      95.5
    ]
  ],
  "winner": "recraft-v3",
  "best_value": "ideogram-v3",
  "correlation": {
    "clip_score_vs_prompt_adherence": {
      "pearson_r": 0.5251,
      "p_value": 0.275198,
      "interpretation": "moderate"
    }
  },
  "cost": {
    "generation": {
      "total_usd": 0.33,
      "by_model": {
        "ideogram-v3": 0.09,
        "recraft-v3": 0.12,
        "flux-pro": 0.12
      },
      "request_count": 9
    },
    "judge": {
      "total_usd": 0.0135,
      "provider": "consensus:gemini-2.5-flash,gpt-5.2",
      "request_count": 27,
      "providers": [
        "gemini-2.5-flash",
        "gpt-5.2"
      ],
      "cost_by_provider": {
        "gemini-2.5-flash": 0.0135,
        "gpt-5.2": 0.0
      }
    },
    "metrics": {
      "total_usd": 0.0,
      "note": "Local computation"
    },
    "total_usd": 0.3435
  },
  "metadata": {
    "evalytic_version": "0.3.11",
    "python_version": "3.14.3",
    "platform": "darwin-arm64"
  },
  "judges": [
    "gemini-2.5-flash",
    "gpt-5.2"
  ],
  "consensus_mode": true
}