<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=1024">
    <title>GenderBench Results</title>
    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/chartjs-plugin-annotation"></script>
    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600&display=swap" rel="stylesheet">
    <script>

        function createChart(canvasId, model_names, intervals, ranges) {

            intervals = intervals.map(item => Array.isArray(item) ? item : [item, item]);
    
            const allPoints = Object.values(ranges).flat().flat();
            const mmin = Math.min(...allPoints);
            const mmax = Math.max(...allPoints);
    
            const ctx = document.getElementById(canvasId).getContext('2d');
    
            const scatter_points = intervals.flatMap(([start, end], index) => [
                { x: start, y: index },
                { x: end, y: index }
            ]).flat();
    
            const data = {
                datasets: [{
                    data: scatter_points,
                    type: 'line',
                    showLine: true,
                    pointRadius: 1,
                    pointBackgroundColor: 'rgba(75, 75, 75, 1)',
                    pointBorderColor: 'rgba(75, 75, 75, 1)',
                    segment: {
                        borderColor: (ctx) => {
                            return ctx.p0.parsed.y === ctx.p1.parsed.y ? 'rgba(75, 75, 75, 1)' : 'transparent';
                        }
                    }
                }]
            };
    
            colors = ["rgb(40, 167, 69, 0.25)", "rgb(255, 193, 7, 0.25)","rgb(253, 126, 20, 0.25)","rgb(220, 53, 69, 0.25)",];
            console.log(ranges)
            const annotations = Object.fromEntries(
                Object.entries(ranges).flatMap(([key, intervals]) =>
                    intervals.map((interval, index) => {
                    const [a, b] = interval;
                    const boxId = `box_${key}_${index}`; // Unique box ID
                    return [
                        boxId,
                        {
                        type: 'box',
                        xMin: a,
                        xMax: b,
                        yMin: -0.5,
                        yMax: model_names.length - 0.5,
                        borderWidth: 0,
                        backgroundColor: colors[key],
                        },
                    ];
                    })
                )
                );
    
            const config = {
                type: 'scatter',
                data: data,
                options: {  
                    responsive: true,
                    maintainAspectRatio: false
                },
                options: {
                    animation: false,
                    scales: {
                        x: {
                            grid: {
                                drawBorder: false,
                                drawOnChartArea: false,
                            },
                            min: mmin,
                            max: mmax,
                            border: {
                                display: false,
                            }
                        },
                        y: {
                            reverse: true,
                            afterBuildTicks: axis => axis.ticks = model_names.map((_, i) => ({ value: i })),
                            ticks: {
                                callback: function(value) {
                                    return model_names[value];
                                },
                            },
                            min: -0.5,
                            max: model_names.length - 0.5,
                            grid: {
                                drawBorder: false,
                            },
                        }
                    },
                    plugins: {
                        legend: {
                            display: false,
                        },
                        annotation: {
                            annotations: annotations
                        }
                    }
                }
            };
    
            const myChart = new Chart(ctx, config);
        }
    </script>
    <style>
        
        body {
            margin: 0;
            font-family: 'Inter', sans-serif;
            background-color: #f8f9fa;
            color: #333;
            line-height: 1.6;
            display: flex;
            flex-direction: column;
            align-items: center;
            padding: 20px;
        }

        
        .container {
            width: 80%;
            max-width: 1000px;
            background-color: #ffffff;
            padding: 20px 30px;
            border-radius: 8px;
            box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
            margin-bottom: 20px;
        }

        h1 {
            font-size: 1.8rem;
            text-align: center;
            margin-bottom: 20px;
        }

        h2 {
            margin: 0;
            font-size: 120%;
        }

        p, ul {
            font-size: 1rem;
            margin-bottom: 30px;
            width: 70%;
        }

        
        #emoji-table1, #emoji-table2 {
            border-collapse: separate;
            border-spacing: 10px; 
            margin-bottom: 20px;
        }

        #emoji-table1 th, #emoji-table2 th {
            text-align: center;
            font-weight: 600;
            padding-bottom: 10px;
        }

        #emoji-table1 td, #emoji-table2 td {
            text-align: center;
            padding: 10px;
        }

        #emoji-table1 {
            display: none;
        }

        #emoji-table2 {
            display: table;
        }

        /* Optional label styling */
        label[for="emojiToggle"] {
            display: inline-block;
            padding: 6px 12px;
            background-color: #eee;
            border: 1px solid #ccc;
            cursor: pointer;
            margin-bottom: 10px;
        }

        .canvas-table {
            margin-top: 20px;
        }

        .canvas-table td {
            padding: 0 15px 0 0px;
        }

        td.mark-A,
        td.mark-B,
        td.mark-C,
        td.mark-D {
            padding: 5px 0;
            font-weight: 600;
            border-radius: 8px;
            color: #ffffff;
            margin: auto; 
            text-align: center;
            font-size: 0.9rem;
            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
            width: 80px;
        }

        strong.mark-A,
        strong.mark-B,
        strong.mark-C,
        strong.mark-D {
            padding: 0 5px;
            font-weight: 600;
            color: #ffffff;
        }

        .mark-A {
            background-color: rgb(40, 167, 69); 
        }

        .mark-B {
            background-color: rgb(255, 193, 7); 
        }

        .mark-C {
            background-color: rgb(253, 126, 20); 
        }

        .mark-D {
            background-color: rgb(220, 53, 69); 
        }

        .canvas-wrapper {
            display: flex; 
            margin-bottom: 50px; 
        }

        canvas {
            width: 90%;
            margin: 0 auto;
        }

        .description {
            flex: 1;
        }

        .details {
            margin: 20px 0;
        }

        hr {
            margin: 20px 0;
        }

        .tag {
            display: inline-block; 
            padding: 8px 12px; 
            background-color: #007bff; 
            color: white; 
            border-radius: 14px; 
            font-size: 10px; 
            font-weight: bold; 
            text-align: center; 
            margin: 10px 10px 10px -3px; 
            cursor: pointer; 
            transition: background-color 0.3s; 
            clear: left;
            padding: 2px 10px;
        }

        #authors {
            text-align: center;
            font-style: italic;
        }

        .normalized-table {
            thead th {
                vertical-align: bottom; 
                span {
                    writing-mode: vertical-rl;
                    transform: rotate(180deg);
                }
            }
            tbody th {
                text-align: right;
                padding: 0 1em;

            }
            margin: 2em auto;
            font-size: 60%;
            border-spacing: 0;
            border: none;
            max-width: 100%;
            th {
                padding: 0.3em;
                border: none;
            }
            td {
                border: none;
                padding: 1em 0.7em;
            }
        }
    </style>
</head>
<body>

    <div class="container">
        <h1>GenderBench {{ version }} Results</h1>
        <div id="authors">Your Name</div>
        <h3>What is GenderBench?</h3>
        <p><em>GenderBench</em> is an open-source evaluation suite designed to comprehensively benchmark <strong>gender biases</strong> in large language models (LLMs). It uses a variety of tests, called <strong>probes</strong>, each targeting a specific type of unfair behavior.</p>
        <h3>What is this document?</h3>
        <p>This document presents the results of <em>GenderBench {{ version }}</em> library, evaluating various LLMs..</p>
        </ul>
        <h3>How can I learn more?</h3>
        <p>For further details, visit the <a href="https://github.com/matus-pikuliak/genderbench">project's repository</a>. We welcome collaborations and contributions.</p> 
    </div>
    <div class="container">
        <h2>Final marks</h2>
        <p>This section presents the main output from our evaluation. Each LLM has received marks based on its performance with various probes. To categorize the severity of harmful behaviors, we use a four-tier system:</p>
        <p>
            <ul>
                <li><strong class="mark-A">A - Healthy.</strong> No detectable signs of harmful behavior.</li>
                <li><strong class="mark-B">B - Cautionary.</strong> Low-intensity harmful behavior, often subtle enough to go unnoticed.</li>
                <li><strong class="mark-C">C - Critical.</strong> Noticeable harmful behavior that may affect user experience.</li>
                <li><strong class="mark-D">D - Catastrophic.</strong> Harmful behavior is common and present in most assessed interactions.</li>
            </ul>
        </p>
        <hr>
        <h3>Harms</h3>
        <p>We categorize the behaviors we quantify based on the type of harm they cause:</p>
        <ul>
            <li><strong>Outcome disparity</strong> - Outcome disparity refers to unfair differences in outcomes across genders. This includes differences in the likelihood of receiving a positive outcome (e.g., loan approval from an AI system) as well as discrepancies in predictive accuracy across genders (e.g., the accuracy of an AI-based medical diagnosis).</li>
            <li><strong>Stereotypical reasoning</strong> - Stereotypical reasoning involves using language that reflects stereotypes (e.g., differences in how AI writes business communication for men versus women), or using stereotypical assumptions during reasoning (e.g., agreeing with stereotypical statements about gender roles). Unlike outcome disparity, this category does not focus on directly measurable outcomes but rather on biased patterns in language and reasoning.</li>
            <li><strong>Representational harms</strong> - Representational harms concern how different genders are portrayed, including issues like under-representation, denigration, etc. In the context of our probes, this category currently only addresses gender balance in generated texts.</li>
        </ul>
        <p>
        <hr>
        <h3>Comprehensive table</h3>
        <p>Below is a table that summarizes all the marks received by the evaluated models. It is also possible to categorize the marks by harm. The marks are sorted by their value.</p>
        <label for="emojiToggle"><input type="checkbox" id="emojiToggle" onchange="
            document.getElementById('emoji-table1').style.display = this.checked ? 'table' : 'none';
            document.getElementById('emoji-table2').style.display = this.checked ? 'none' : 'table';
        "> Categorize by harm</label>
        <table id="emoji-table1">
            <thead>
                <tr>
                    <th></th>
                    <th>Outcome disparity</th>
                    <th>Stereotypical reasoning</th>
                    <th>Representational harms</th>
                </tr>
            </thead>
            <tbody>
                {% for row in emoji_table_1 %}
                <tr>
                    {% for item in row %}
                        <td>{{ item }}</td>
                    {% endfor %}
                </tr>
                {% endfor %}
            </tbody>
        </table>
        <table id="emoji-table2">
            <thead>
                <tr>
                    <th></th>
                    <th>All</th>
                </tr>
            </thead>
            <tbody>
                {% for row in emoji_table_2 %}
                <tr>
                    {% for item in row %}
                        <td>{{ item }}</td>
                    {% endfor %}
                </tr>
                {% endfor %}
            </tbody>
        </table>
    </div>

    {% set chart_count = namespace(value=0) %}
    <div class="container">
        <h2>Outcome disparity</h2>
        <p>This section shows the probe results for the outcome disparity probes. This includes differences in the likelihood of receiving a positive outcome (e.g., loan approval from an AI system) as well as discrepancies in predictive accuracy across genders (e.g., the accuracy of an AI-based medical diagnosis).</p>
        <hr>
        {{rendered_sections.outcome_disparity}}
    </div>
    <div class="container">
        <h2>Stereotypical reasoning</h2>
        <p>This section shows the probe results for the stereotypical reasoning probes. Stereotypical reasoning involves using language that reflects stereotypes (e.g., differences in how AI writes business communication for men versus women), or using stereotypical assumptions during reasoning (e.g., agreeing with stereotypical statements about gender roles).</p>
        <hr>
        {{rendered_sections.stereotypical_reasoning}}
    </div>
    <div class="container">
        <h2>Representational harms</h2>
        <p>This section shows the probe results for the representational harms probes. Representational harms concern how different genders are portrayed, including issues like under-representation, denigration, etc.</p>
        <hr>
        {{rendered_sections.representational_harms}}
    </div>
    <div class="container">
        <h2>Treatment of women and men</h2>
        <p>This section directly compares the treatment of men and women in situations when it can clearly be said that one or the other group is being preferred. In the probe below, negative values mean that the LLMs give preferential treatment for women, positive values mean preferential treatment for men.</p>
        <hr>
        {{rendered_sections.mvf}}

    </div>
    <div class="container">
        <h2>Normalized results</h2>
        The table below presents the results used to calculate the marks, normalized in different ways to fall within the [0, 1] interval, where 0 and 1 represent the theoretically least and most biased models respectively. We also display the <em>average</em> result for each model.
        <hr>
        {{normalized_table}}
    </div>
    <div class="container">
        <h2>Methodological Notes</h2>
        <ul>
            <li>The results were obtained by using <a href="https://pypi.org/project/genderbench/">genderbench</a> library version {{ version }}.</li>
            <li>Marks (A-D) are assigned by comparing confidence intervals to predefined thresholds. A probe's final mark is the healthiest category that overlaps with its confidence interval.</li>
        </ul>
    </div>
</body>
</html>
