{
  "$schema": "https://evalytic.ai/schemas/bench-report-v1.json",
  "version": "1.0",
  "name": "leaderboard-pilot-smoke",
  "created_at": "2026-03-12T07:32:36.593297+00:00",
  "duration_seconds": 35.4,
  "config": {
    "models": [
      "flux-schnell",
      "z-image-turbo"
    ],
    "judge": "gemini-2.5-flash",
    "dimensions": [
      "visual_quality",
      "prompt_adherence",
      "text_rendering"
    ],
    "metrics": [
      "clip",
      "sharpness"
    ],
    "pipeline": "text2img",
    "seed": 42,
    "image_size": null,
    "concurrency": 8,
    "scoring_config": {
      "clip_score": {
        "flag_threshold": 0.18,
        "weight": 0.2,
        "normalize_range": [
          0.18,
          0.35
        ]
      },
      "lpips": {
        "flag_threshold": 0.4,
        "weight": 0.2,
        "normalize_range": [
          0.4,
          0.95
        ]
      },
      "face_similarity": {
        "flag_threshold": 0.6,
        "weight": 0.2,
        "normalize_range": [
          0.6,
          0.95
        ]
      }
    }
  },
  "items": [
    {
      "item_id": "item-001",
      "prompt": "A glowing neon sign reading EVALYTIC in a dark rainy alley, reflections on wet cobblestones, cinematic lighting",
      "results": {
        "flux-schnell": {
          "image_url": "https://v3b.fal.media/files/b/0a91d798/fu7vbHMEdCROBl6IIxZZO.jpg",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-pilot-smoke/flux-schnell/item-001.jpg",
          "generation_time_ms": 1354,
          "generation_cost_usd": 0.003,
          "scores": {
            "visual_quality": {
              "score": 4.0,
              "confidence": 0.9,
              "explanation": "The image exhibits high visual quality with excellent atmospheric rendering, realistic lighting, and convincing reflections on the wet ground. The neon sign is sharp and legible, creating a strong focal point and contributing to a cohesive scene. The only minor flaw is some illegible text on a sign on the right building.",
              "evidence": [
                "Sharp and legible neon sign 'EVALYTIC' with realistic reflections on the wet ground.",
                "Consistent and atmospheric lighting, effectively conveying a rainy night scene.",
                "Minor artifact of illegible, gibberish text on a sign on the right building."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly matches all aspects of the prompt, featuring a clearly legible neon sign 'EVALYTIC' in a dark, rainy alley. The reflections on the wet cobblestones are prominent and well-rendered, contributing to the overall cinematic lighting and atmosphere.",
              "evidence": [
                "The glowing neon sign 'EVALYTIC' is clearly visible and legible.",
                "The setting is a dark, rainy alley with visible rain effects and a somber mood.",
                "Strong, accurate reflections of the neon sign and other lights are present on the wet cobblestones.",
                "The lighting is dramatic and atmospheric, consistent with 'cinematic lighting'."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered on the neon sign. The letters are crisp, clear, and accurately spelled, with no visible rendering issues or artifacts.",
              "evidence": [
                "The word 'EVALYTIC' is clearly legible and correctly spelled.",
                "Each letter is distinct and well-formed, exhibiting a consistent neon glow effect.",
                "The reflection of the text on the wet cobblestones is also accurate and readable."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3127,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9446,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            }
          },
          "overall_score": 4.666666666666667,
          "status": "success"
        },
        "z-image-turbo": {
          "image_url": "https://v3b.fal.media/files/b/0a91d798/oEifh8pAQUJKe_0oMtAck_tBTUYqY5.png",
          "image_local": "/Users/jangis/.evalytic/cache/leaderboard-pilot-smoke/z-image-turbo/item-001.png",
          "generation_time_ms": 2204,
          "generation_cost_usd": 0.005,
          "scores": {
            "visual_quality": {
              "score": 5.0,
              "confidence": 0.95,
              "explanation": "The image exhibits excellent visual quality, effectively capturing a moody, rainy night scene with realistic lighting and atmospheric effects. The neon sign is perfectly rendered, and its reflections on the wet cobblestones are highly convincing, contributing to a professional-grade aesthetic.",
              "evidence": [
                "Sharp and perfectly legible neon text 'EVALYTIC' with a realistic, consistent glow.",
                "Highly convincing reflections of the neon light and ambient colors on the wet cobblestone ground, accurately depicting light interaction with water.",
                "Consistent and atmospheric rendering of falling rain and wet surfaces, enhancing the scene's realism.",
                "No visible artifacts, blurring, or incoherent elements are present throughout the image."
              ]
            },
            "prompt_adherence": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The image perfectly captures every element described in the prompt. The neon sign clearly reads 'EVALYTIC' and glows brightly in a dark, rainy alley, with excellent reflections on the wet cobblestones and a strong cinematic atmosphere.",
              "evidence": [
                "The neon sign accurately displays 'EVALYTIC' and is glowing.",
                "The setting is a dark, rainy alley with visible rain streaks.",
                "The cobblestones are wet and show clear, vibrant reflections of the neon light.",
                "The overall lighting and mood are highly cinematic, with strong contrast and atmospheric effects."
              ]
            },
            "text_rendering": {
              "score": 5.0,
              "confidence": 1.0,
              "explanation": "The text 'EVALYTIC' is perfectly rendered on the neon sign. It is crisp, accurate, and clearly legible with no spelling errors or visual artifacts.",
              "evidence": [
                "The word 'EVALYTIC' is spelled correctly.",
                "Each letter is distinct and well-formed, exhibiting a realistic neon glow.",
                "There are no distortions or unreadable parts in the text."
              ]
            }
          },
          "metrics": {
            "clip_score": {
              "value": 0.3581,
              "description": "CLIP cosine similarity (text-image)"
            },
            "sharpness": {
              "value": 0.9432,
              "description": "Variance of Laplacian sharpness (0-1, higher=sharper)"
            }
          },
          "overall_score": 5.0,
          "status": "success"
        }
      }
    }
  ],
  "summary": {
    "flux-schnell": {
      "overall_score": 4.52,
      "dimension_averages": {
        "visual_quality": 4.0,
        "prompt_adherence": 5.0,
        "text_rendering": 5.0
      },
      "dimension_confidence": {
        "visual_quality": 0.9,
        "prompt_adherence": 1.0,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.97,
      "metric_averages": {
        "clip_score": 0.3127,
        "sharpness": 0.9446
      },
      "total_generation_cost_usd": 0.003,
      "total_generation_time_ms": 1354,
      "cost_per_image": 0.003,
      "score_per_dollar": 1506.7,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.0
      },
      "overall_stddev": 0.47,
      "sample_count": 1,
      "total_items": 1,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included"
      },
      "weighted_overall_score": 4.52
    },
    "z-image-turbo": {
      "overall_score": 5.0,
      "dimension_averages": {
        "visual_quality": 5.0,
        "prompt_adherence": 5.0,
        "text_rendering": 5.0
      },
      "dimension_confidence": {
        "visual_quality": 0.95,
        "prompt_adherence": 1.0,
        "text_rendering": 1.0
      },
      "avg_confidence": 0.98,
      "metric_averages": {
        "clip_score": 0.3581,
        "sharpness": 0.9432
      },
      "total_generation_cost_usd": 0.005,
      "total_generation_time_ms": 2204,
      "cost_per_image": 0.005,
      "score_per_dollar": 1000.0,
      "dimension_stddev": {
        "visual_quality": 0.0,
        "prompt_adherence": 0.0,
        "text_rendering": 0.0
      },
      "sample_count": 1,
      "total_items": 1,
      "success_rate": 1.0,
      "metric_flags": {
        "clip_score": "included"
      },
      "weighted_overall_score": 5.0
    }
  },
  "ranking": [
    [
      "z-image-turbo",
      5.0
    ],
    [
      "flux-schnell",
      4.52
    ]
  ],
  "efficiency_ranking": [
    [
      "flux-schnell",
      1506.7
    ],
    [
      "z-image-turbo",
      1000.0
    ]
  ],
  "winner": "z-image-turbo",
  "best_value": "flux-schnell",
  "correlation": {},
  "cost": {
    "generation": {
      "total_usd": 0.008,
      "by_model": {
        "flux-schnell": 0.003,
        "z-image-turbo": 0.005
      },
      "request_count": 2
    },
    "judge": {
      "total_usd": 0.003,
      "provider": "gemini-2.5-flash",
      "request_count": 6
    },
    "metrics": {
      "total_usd": 0.0,
      "note": "Local computation"
    },
    "total_usd": 0.011
  },
  "metadata": {
    "evalytic_version": "0.3.10",
    "python_version": "3.14.3",
    "platform": "darwin-arm64"
  }
}