{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "colab": {
      "name": "compas_playground_multiframework.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xK1E-V3KJLUD"
      },
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/mparrott-at-wiris/aimodelshare/blob/master/notebooks/notebooks_compas_playground_multiframework.ipynb)\n",
        "\n",
        "# COMPAS Multi-Framework Playground (Lightweight)\n",
        "This notebook replicates (in interactive form) the lightweight CI test logic found in `tests/test_playground_compas_multiframework_short.py`.\n",
        "\n",
        "It will:\n",
        "1. Configure `aimodelshare` credentials (interactive prompt).\n",
        "2. Load & preprocess the ProPublica COMPAS two-year recidivism dataset (sampled to 2,500 rows for efficiency).\n",
        "3. Create a private (optionally public) `ModelPlayground` for a classification task.\n",
        "4. Train and submit minimal models across frameworks:\n",
        "   - Scikit-learn: Logistic Regression, Random Forest\n",
        "   - Keras: Simple Sequential Dense Network\n",
        "   - PyTorch: Basic MLP\n",
        "5. Attach custom metadata field `Moral_Compass_Fairness` cycling through values (0.25, 0.50, 0.75) per submission.\n",
        "6. Display leaderboard and validate tag presence.\n",
        "\n",
        "Run cells in order. If ONNX or stdin-related export issues occur (sometimes in constrained environments), those submissions are skipped gracefully.\n",
        "\n",
        "**NOTE:** For real usage, ensure you have valid AWS and platform credentials. In Colab you may paste them directly when prompted.\n"
      ],
      "id": "xK1E-V3KJLUD"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "APdfjhuiJLUI"
      },
      "source": [
        "## 1. Install / Upgrade Dependencies\n",
        "If running in a fresh Colab environment, install (or upgrade) the required packages."
      ],
      "id": "APdfjhuiJLUI"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "install-deps",
        "outputId": "026feef3-d6c9-497c-89fd-cd17a564418b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m998.2/998.2 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m57.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m352.5/352.5 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.4/17.4 MB\u001b[0m \u001b[31m57.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m165.8/165.8 kB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m315.5/315.5 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m451.2/451.2 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m63.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.7/85.7 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.19.0 requires flatbuffers>=24.3.25, but you have flatbuffers 2.0.7 which is incompatible.\n",
            "firebase-admin 6.9.0 requires pyjwt[crypto]>=2.5.0, but you have pyjwt 1.7.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "!pip install --quiet aimodelshare"
      ],
      "id": "install-deps"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMr7V_29JLUK"
      },
      "source": [
        "## 2. Imports & Global Configuration\n",
        "Set random seeds for reproducibility; define constants & feature lists matching the test script."
      ],
      "id": "uMr7V_29JLUK"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imports",
        "outputId": "0105d905-9028-4609-d09e-65d2e126e794",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Imports and globals initialized.\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "import itertools\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import requests\n",
        "from io import StringIO\n",
        "from getpass import getpass\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers, Sequential\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from aimodelshare.playground import ModelPlayground\n",
        "from aimodelshare.aws import configure_credentials, set_credentials, get_aws_token\n",
        "from aimodelshare.modeluser import get_jwt_token, create_user_getkeyandpassword\n",
        "\n",
        "# Moral Compass imports added later for challenge section\n",
        "\n",
        "# Reproducibility\n",
        "np.random.seed(42)\n",
        "tf.random.set_seed(42)\n",
        "torch.manual_seed(42)\n",
        "\n",
        "# Dataset configuration\n",
        "MAX_ROWS = 2500\n",
        "TOP_N_CHARGE_CATEGORIES = 50\n",
        "\n",
        "# Feature sets (align with test file constants)\n",
        "NUMERIC_FEATURES = ['age', 'priors_count', 'juv_fel_count', 'juv_misd_count', 'juv_other_count', 'days_b_screening_arrest']\n",
        "CATEGORICAL_FEATURES = ['race', 'sex', 'age_cat', 'c_charge_degree', 'c_charge_desc']\n",
        "\n",
        "def fairness_value_generator():\n",
        "    \"\"\"Cycle through fairness values for custom metadata submissions.\"\"\"\n",
        "    return itertools.cycle([0.25, 0.50, 0.75])\n",
        "\n",
        "def build_custom_metadata(fairness_value: float) -> dict:\n",
        "    return {\"Moral_Compass_Fairness\": f\"{fairness_value:.2f}\"}\n",
        "\n",
        "print(\"Imports and globals initialized.\")"
      ],
      "id": "imports"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "naKx1YjLJLUL"
      },
      "source": [
        "## 3. Configure Credentials\n",
        "You will be prompted for:\n",
        "- Platform username & password\n",
        "- AWS Access Key ID\n",
        "- AWS Secret Access Key\n",
        "- AWS Region\n",
        "\n",
        "They will be stored temporarily in a local `credentials.txt` file and set for deployment. If you've already configured credentials in this environment you may skip re-running.\n"
      ],
      "id": "naKx1YjLJLUL"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "credentials",
        "outputId": "0ba0d283-4e54-4654-ab1f-544bd4985534",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Configure aimodelshare credentials (follow prompts):\n",
            "Modelshare.ai Username:··········\n",
            "Modelshare.ai Password:··········\n",
            "AWS_ACCESS_KEY_ID:··········\n",
            "AWS_SECRET_ACCESS_KEY:··········\n",
            "AWS_REGION:··········\n",
            "Configuration successful. New credentials file saved as 'credentials.txt'\n",
            "Modelshare.ai login credentials set successfully.\n",
            "AWS credentials set successfully.\n",
            "AWS token retrieved.\n",
            "JWT validation succeeded.\n",
            "Credentials configured.\n"
          ]
        }
      ],
      "source": [
        "print(\"Configure aimodelshare credentials (follow prompts):\")\n",
        "configure_credentials()  # interactive prompts\n",
        "set_credentials(credential_file=\"credentials.txt\", type=\"deploy_model\")\n",
        "\n",
        "try:\n",
        "    # Attempt to acquire tokens (optional validation)\n",
        "    aws_token = get_aws_token()\n",
        "    print(\"AWS token retrieved.\")\n",
        "except Exception as e:\n",
        "    print(f\"Warning: Could not retrieve AWS token: {e}\")\n",
        "\n",
        "try:\n",
        "    # Validate JWT tokens; create user key/password if needed\n",
        "    username = os.environ.get('username') or input(\"Re-enter platform username (for JWT test): \")\n",
        "    password = os.environ.get('password') or getpass(\"Re-enter platform password (hidden): \")\n",
        "    get_jwt_token(username, password)\n",
        "    create_user_getkeyandpassword()\n",
        "    print(\"JWT validation succeeded.\")\n",
        "except Exception as e:\n",
        "    print(f\"Warning: JWT validation issue: {e}\")\n",
        "\n",
        "print(\"Credentials configured.\")"
      ],
      "id": "credentials"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ey5cAX-vJLUM"
      },
      "source": [
        "## 4. Load & Preprocess COMPAS Data\n",
        "Mirrors logic from the test file: download, sample, feature engineer charge description categories, split, and construct preprocessing pipeline."
      ],
      "id": "ey5cAX-vJLUM"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "data-prep",
        "outputId": "6056ac5d-dc05-412b-aab9-b91224299584",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloaded COMPAS dataset: (7214, 53)\n",
            "Sampled to 2500 rows\n",
            "Features shape: (2500, 11); Target distribution: {0: 1422, 1: 1078}\n",
            "Train shape: (1875, 11); Test shape: (625, 11)\n",
            "Preprocessing pipeline fitted.\n"
          ]
        }
      ],
      "source": [
        "COMPAS_URL = \"https://raw.githubusercontent.com/propublica/compas-analysis/master/compas-scores-two-years.csv\"\n",
        "response = requests.get(COMPAS_URL)\n",
        "df = pd.read_csv(StringIO(response.text))\n",
        "print(f\"Downloaded COMPAS dataset: {df.shape}\")\n",
        "\n",
        "# Sample for performance\n",
        "if df.shape[0] > MAX_ROWS:\n",
        "    df = df.sample(n=MAX_ROWS, random_state=42)\n",
        "    print(f\"Sampled to {MAX_ROWS} rows\")\n",
        "\n",
        "feature_columns = [\n",
        "    'race', 'sex', 'age', 'age_cat', 'c_charge_degree', 'c_charge_desc',\n",
        "    'priors_count', 'juv_fel_count', 'juv_misd_count', 'juv_other_count', 'days_b_screening_arrest'\n",
        "]\n",
        "target_column = 'two_year_recid'\n",
        "\n",
        "# Condense c_charge_desc to top-N + OTHER_DESC\n",
        "if 'c_charge_desc' in df.columns:\n",
        "    top_charges = df['c_charge_desc'].value_counts().head(TOP_N_CHARGE_CATEGORIES).index\n",
        "    df['c_charge_desc'] = df['c_charge_desc'].apply(\n",
        "        lambda x: x if pd.notna(x) and x in top_charges else 'OTHER_DESC'\n",
        "    )\n",
        "\n",
        "X = df[feature_columns].copy()\n",
        "y = df[target_column].values\n",
        "print(f\"Features shape: {X.shape}; Target distribution: {pd.Series(y).value_counts().to_dict()}\")\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.25, random_state=42, stratify=y\n",
        ")\n",
        "print(f\"Train shape: {X_train.shape}; Test shape: {X_test.shape}\")\n",
        "\n",
        "numeric_transformer = Pipeline(steps=[\n",
        "    ('imputer', SimpleImputer(strategy='median')),\n",
        "    ('scaler', StandardScaler())\n",
        "])\n",
        "\n",
        "categorical_transformer = Pipeline(steps=[\n",
        "    ('imputer', SimpleImputer(strategy='most_frequent')),\n",
        "    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))\n",
        "])\n",
        "\n",
        "preprocessor = ColumnTransformer(transformers=[\n",
        "    ('num', numeric_transformer, NUMERIC_FEATURES),\n",
        "    ('cat', categorical_transformer, CATEGORICAL_FEATURES)\n",
        "])\n",
        "\n",
        "preprocessor.fit(X_train)\n",
        "\n",
        "def preprocessor_func(data):\n",
        "    return preprocessor.transform(data)\n",
        "\n",
        "print(\"Preprocessing pipeline fitted.\")"
      ],
      "id": "data-prep"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9TP_Gt40JLUN"
      },
      "source": [
        "## 5. Create ModelPlayground\n",
        "We create a classification playground, using the test labels as evaluation data. Set `public=True` if you want it discoverable (optional)."
      ],
      "id": "9TP_Gt40JLUN"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "playground-create",
        "outputId": "c60a5ccb-9afb-42e9-c9ac-0dd76cccbeae",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Creating your prediction API. (This process may take several minutes.)\n",
            "\n",
            "[=====================================] Progress: 100% - Complete!                                            \n",
            "\n",
            "Success! Your Model Playground was created in 53 seconds. \n",
            " Playground Url: \"https://pk8xla3k89.execute-api.us-east-1.amazonaws.com/prod/m\"\n",
            "\n",
            "You can now use your Model Playground.\n",
            "\n",
            "Follow this link to explore your Model Playground's functionality\n",
            "You can make predictions with the Dashboard and access example code from the Programmatic tab.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "\n",
            "Check out your Model Playground page for more.\n",
            "Playground created.\n",
            "Playground ID: None\n"
          ]
        }
      ],
      "source": [
        "eval_labels = list(y_test)\n",
        "playground = ModelPlayground(input_type='tabular', task_type='classification', private=True)\n",
        "playground.create(eval_data=eval_labels, public=True)\n",
        "print(\"Playground created.\")\n",
        "try:\n",
        "    playground_id = getattr(playground, 'playground_id', None) or getattr(playground, 'id', None)\n",
        "    print(f\"Playground ID: {playground_id}\")\n",
        "except Exception:\n",
        "    playground_id = None\n",
        "    print(\"Could not access playground ID attribute.\")"
      ],
      "id": "playground-create"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LO0OSfFhJLUO"
      },
      "source": [
        "## 6. Helper Function for Submissions\n",
        "Handles metadata, PyTorch dummy input creation, and ONNX/stdin error skipping."
      ],
      "id": "LO0OSfFhJLUO"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "helper-function"
      },
      "execution_count": 6,
      "outputs": [],
      "source": [
        "def submit_model_helper(playground, model, preprocessor_obj, preds, framework, model_name, submission_type, fairness_value):\n",
        "    try:\n",
        "        extra_kwargs = {}\n",
        "        if framework == 'pytorch':\n",
        "            # Build dummy input after preprocessing a single synthetic row\n",
        "            dummy_data = {feat: [0] for feat in NUMERIC_FEATURES}\n",
        "            dummy_data.update({feat: ['A'] for feat in CATEGORICAL_FEATURES})\n",
        "            X_dummy = pd.DataFrame(dummy_data)\n",
        "            X_processed = preprocessor_obj.transform(X_dummy)\n",
        "            input_dim = X_processed.shape[1]\n",
        "            dummy_input = torch.zeros((1, input_dim), dtype=torch.float32)\n",
        "            extra_kwargs['model_input'] = dummy_input\n",
        "\n",
        "        custom_metadata = build_custom_metadata(fairness_value)\n",
        "        print(f\"Submitting {model_name} ({framework}) as {submission_type} with metadata: {custom_metadata}\")\n",
        "\n",
        "        playground.submit_model(\n",
        "            model=model,\n",
        "            preprocessor=preprocessor_obj,\n",
        "            prediction_submission=preds,\n",
        "            input_dict={\n",
        "                'description': f'Notebook submission {framework} {model_name} COMPAS_short {submission_type}',\n",
        "                'tags': f'compas_short,{framework},{submission_type}'\n",
        "            },\n",
        "            submission_type=submission_type,\n",
        "            custom_metadata=custom_metadata,\n",
        "            **extra_kwargs\n",
        "        )\n",
        "        print(\"✓ Submission succeeded.\")\n",
        "        return True\n",
        "    except Exception as e:\n",
        "        error_lower = str(e).lower()\n",
        "        if 'stdin' in error_lower or 'onnx' in error_lower:\n",
        "            print(f\"⊘ Skipped {model_name} due to ONNX/stdin export issue: {e}\")\n",
        "            return False\n",
        "        print(f\"✗ Submission failed: {e}\")\n",
        "        return False"
      ],
      "id": "helper-function"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kreKkgCeJLUO"
      },
      "source": [
        "## 7. Train & Submit Scikit-learn Models"
      ],
      "id": "kreKkgCeJLUO"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sklearn-submit",
        "outputId": "2bf18f45-14f4-4c6f-d79a-977d4fe2e421",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "------------------------------------------------------------\n",
            "Training LogisticRegression\n",
            "Predictions generated: 625; Distribution: {0: 335, 1: 290}\n",
            "Submitting LogisticRegression (sklearn) as competition with metadata: {'Moral_Compass_Fairness': '0.25'}\n",
            "\n",
            "Your model has been submitted to competition as model version 1.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n",
            "Submitting LogisticRegression (sklearn) as experiment with metadata: {'Moral_Compass_Fairness': '0.50'}\n",
            "\n",
            "Your model has been submitted to experiment as model version 1.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n",
            "\n",
            "------------------------------------------------------------\n",
            "Training RandomForestClassifier\n",
            "Predictions generated: 625; Distribution: {0: 369, 1: 256}\n",
            "Submitting RandomForestClassifier (sklearn) as competition with metadata: {'Moral_Compass_Fairness': '0.75'}\n",
            "\n",
            "Your model has been submitted to competition as model version 2.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n",
            "Submitting RandomForestClassifier (sklearn) as experiment with metadata: {'Moral_Compass_Fairness': '0.25'}\n",
            "\n",
            "Your model has been submitted to experiment as model version 2.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n"
          ]
        }
      ],
      "source": [
        "fairness_gen = fairness_value_generator()\n",
        "\n",
        "X_train_processed = preprocessor_func(X_train)\n",
        "X_test_processed = preprocessor_func(X_test)\n",
        "\n",
        "sklearn_models = [\n",
        "    (\"LogisticRegression\", LogisticRegression(max_iter=500, random_state=42, class_weight='balanced')),\n",
        "    (\"RandomForestClassifier\", RandomForestClassifier(n_estimators=40, max_depth=10, random_state=42, class_weight='balanced')),\n",
        "]\n",
        "\n",
        "for name, model in sklearn_models:\n",
        "    print(\"\\n\" + \"-\"*60)\n",
        "    print(f\"Training {name}\")\n",
        "    try:\n",
        "        model.fit(X_train_processed, y_train)\n",
        "        if hasattr(model, 'predict_proba'):\n",
        "            proba = model.predict_proba(X_test_processed)[:, 1]\n",
        "            preds = (proba >= 0.5).astype(int)\n",
        "        else:\n",
        "            preds = model.predict(X_test_processed)\n",
        "        print(f\"Predictions generated: {len(preds)}; Distribution: {pd.Series(preds).value_counts().to_dict()}\")\n",
        "    except Exception as e:\n",
        "        print(f\"✗ Training failed for {name}: {e}\")\n",
        "        continue\n",
        "\n",
        "    for submission_type in ['competition', 'experiment']:\n",
        "        fairness_val = next(fairness_gen)\n",
        "        submit_model_helper(playground, model, preprocessor, preds, 'sklearn', name, submission_type, fairness_val)"
      ],
      "id": "sklearn-submit"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A1PpgdkbJLUP"
      },
      "source": [
        "## 8. Train & Submit Keras Model"
      ],
      "id": "A1PpgdkbJLUP"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "keras-submit",
        "outputId": "6ca22b9b-27bc-4ea7-fc9c-922b87af91e5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "============================================================\n",
            "Training Keras Sequential Model\n",
            "Keras predictions distribution: {0: 406, 1: 219}\n",
            "Submitting sequential_dense (keras) as competition with metadata: {'Moral_Compass_Fairness': '0.50'}\n",
            "\n",
            "Your model has been submitted to competition as model version 3.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n",
            "Submitting sequential_dense (keras) as experiment with metadata: {'Moral_Compass_Fairness': '0.75'}\n",
            "\n",
            "Your model has been submitted to experiment as model version 3.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n\" + \"=\"*60)\n",
        "print(\"Training Keras Sequential Model\")\n",
        "input_dim = X_train_processed.shape[1]\n",
        "\n",
        "keras_model = Sequential([\n",
        "    layers.Input(shape=(input_dim,)),\n",
        "    layers.Dense(64, activation='relu'),\n",
        "    layers.Dense(32, activation='relu'),\n",
        "    layers.Dense(1, activation='sigmoid')\n",
        "])\n",
        "keras_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "try:\n",
        "    keras_model.fit(X_train_processed, y_train, epochs=6, batch_size=64, verbose=0, validation_split=0.1)\n",
        "    proba = keras_model.predict(X_test_processed, verbose=0).flatten()\n",
        "    keras_preds = (proba >= 0.5).astype(int)\n",
        "    print(f\"Keras predictions distribution: {pd.Series(keras_preds).value_counts().to_dict()}\")\n",
        "    for submission_type in ['competition', 'experiment']:\n",
        "        fairness_val = next(fairness_gen)\n",
        "        submit_model_helper(playground, keras_model, preprocessor, keras_preds, 'keras', 'sequential_dense', submission_type, fairness_val)\n",
        "except Exception as e:\n",
        "    print(f\"✗ Keras training/submission failed: {e}\")"
      ],
      "id": "keras-submit"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_oYaxL8IJLUP"
      },
      "source": [
        "## 9. Train & Submit PyTorch Model"
      ],
      "id": "_oYaxL8IJLUP"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pytorch-submit",
        "outputId": "516dbdb8-e13b-4d8f-bfef-a28cc4c8732a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "============================================================\n",
            "Training PyTorch MLP Model\n",
            "PyTorch predictions distribution: {0: 400, 1: 225}\n",
            "Submitting mlp_basic (pytorch) as competition with metadata: {'Moral_Compass_Fairness': '0.25'}\n",
            "\n",
            "Your model has been submitted to competition as model version 4.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n",
            "Submitting mlp_basic (pytorch) as experiment with metadata: {'Moral_Compass_Fairness': '0.50'}\n",
            "\n",
            "Your model has been submitted to experiment as model version 4.\n",
            "\n",
            "Visit your Model Playground Page for more.\n",
            "https://www.modelshare.ai/detail/model:4105\n",
            "✓ Submission succeeded.\n"
          ]
        }
      ],
      "source": [
        "print(\"\\n\" + \"=\"*60)\n",
        "print(\"Training PyTorch MLP Model\")\n",
        "input_dim = X_train_processed.shape[1]\n",
        "\n",
        "class MLPBasic(nn.Module):\n",
        "    def __init__(self, input_size):\n",
        "        super().__init__()\n",
        "        self.fc1 = nn.Linear(input_size, 64)\n",
        "        self.fc2 = nn.Linear(64, 32)\n",
        "        self.fc3 = nn.Linear(32, 1)\n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.fc1(x))\n",
        "        x = F.relu(self.fc2(x))\n",
        "        x = self.fc3(x)\n",
        "        return x\n",
        "\n",
        "pytorch_model = MLPBasic(input_dim)\n",
        "\n",
        "try:\n",
        "    X_train_tensor = torch.FloatTensor(X_train_processed)\n",
        "    y_train_tensor = torch.FloatTensor(y_train).unsqueeze(1)\n",
        "    X_test_tensor = torch.FloatTensor(X_test_processed)\n",
        "\n",
        "    criterion = nn.BCEWithLogitsLoss()\n",
        "    optimizer = torch.optim.Adam(pytorch_model.parameters(), lr=0.001)\n",
        "\n",
        "    dataset = torch.utils.data.TensorDataset(X_train_tensor, y_train_tensor)\n",
        "    dataloader = torch.utils.data.DataLoader(dataset, batch_size=64, shuffle=True)\n",
        "\n",
        "    pytorch_model.train()\n",
        "    for epoch in range(6):\n",
        "        for batch_X, batch_y in dataloader:\n",
        "            optimizer.zero_grad()\n",
        "            outputs = pytorch_model(batch_X)\n",
        "            loss = criterion(outputs, batch_y)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "    pytorch_model.eval()\n",
        "    with torch.no_grad():\n",
        "        logits = pytorch_model(X_test_tensor)\n",
        "        proba = torch.sigmoid(logits).numpy().flatten()\n",
        "        pytorch_preds = (proba >= 0.5).astype(int)\n",
        "    print(f\"PyTorch predictions distribution: {pd.Series(pytorch_preds).value_counts().to_dict()}\")\n",
        "\n",
        "    for submission_type in ['competition', 'experiment']:\n",
        "        fairness_val = next(fairness_gen)\n",
        "        submit_model_helper(playground, pytorch_model, preprocessor, pytorch_preds, 'pytorch', 'mlp_basic', submission_type, fairness_val)\n",
        "except Exception as e:\n",
        "    print(f\"✗ PyTorch training/submission failed: {e}\")"
      ],
      "id": "pytorch-submit"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0LxMGnndJLUP"
      },
      "source": [
        "## 10. Retrieve & Inspect Leaderboard\n",
        "Ensures submissions exist and prints full leaderboard for review."
      ],
      "id": "0LxMGnndJLUP"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "leaderboard",
        "outputId": "c5ffb946-6163-4e93-abbc-a0ac24b5a8de",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Leaderboard entries: 4\n",
            " accuracy  f1_score  precision   recall ml_framework             model_type  depth  num_params  dense_layers  sigmoid_act  relu_act loss optimizer  Moral_Compass_Fairness     username                  timestamp  version\n",
            "   0.6944  0.686371   0.687960 0.685409      sklearn RandomForestClassifier    NaN         NaN           NaN          NaN       NaN  NaN       NaN                    0.25 mikedparrott 2025-10-29 08:56:34.742668        2\n",
            "   0.6928  0.678684   0.688056 0.676737      pytorch             MLPBasic()    NaN         NaN           NaN          NaN       NaN  NaN       NaN                    0.50 mikedparrott 2025-10-29 08:57:18.704495        4\n",
            "   0.6752  0.671537   0.671076 0.673552      sklearn     LogisticRegression    NaN        70.0           NaN          NaN       NaN  NaN     lbfgs                    0.50 mikedparrott 2025-10-29 08:56:22.191170        1\n",
            "   0.6832  0.667210   0.678341 0.665584        keras             Sequential    3.0      6657.0           3.0          1.0       2.0  str      Adam                    0.75 mikedparrott 2025-10-29 08:56:59.816616        3\n"
          ]
        }
      ],
      "source": [
        "try:\n",
        "    lb_data = playground.get_leaderboard()\n",
        "    if isinstance(lb_data, dict):\n",
        "        df_lb = pd.DataFrame(lb_data)\n",
        "    else:\n",
        "        df_lb = lb_data\n",
        "\n",
        "    if df_lb.empty:\n",
        "        print(\"Leaderboard is empty. (Possibly all submissions failed or were skipped.)\")\n",
        "    else:\n",
        "        print(f\"Leaderboard entries: {len(df_lb)}\")\n",
        "        if 'tags' in df_lb.columns:\n",
        "            tag_series = df_lb['tags'].astype(str)\n",
        "            print(\"Tag counts:\")\n",
        "            for t in ['compas_short', 'sklearn', 'keras', 'pytorch', 'competition', 'experiment']:\n",
        "                print(f\"  {t}: {tag_series.str.contains(t, case=False, na=False).sum()}\")\n",
        "\n",
        "        with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 1200):\n",
        "            print(df_lb.to_string(index=False))\n",
        "except Exception as e:\n",
        "    print(f\"Failed to retrieve leaderboard: {e}\")"
      ],
      "id": "leaderboard"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0FdVC3oaJLUQ"
      },
      "source": [
        "## 11. Next Steps\n",
        "- Adjust model architectures or hyperparameters for experimentation.\n",
        "- Use different fairness metadata strategies.\n",
        "- Toggle playground visibility or extend with new frameworks.\n",
        "- Integrate automated evaluation workflows.\n",
        "\n",
        "This concludes the lightweight multi-framework COMPAS submission demo."
      ],
      "id": "0FdVC3oaJLUQ"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cg_bl78bJLUQ"
      },
      "source": [
        "---\n",
        "# Moral Compass Challenge Extension\n",
        "The following sections extend the notebook to test creation of a new Moral Compass challenge table and simulate a user progressing through tasks and questions to obtain a final Moral Compass score.\n",
        "\n",
        "Logic adapted from `tests/test_playground_moral_compass_challenge.py`."
      ],
      "id": "cg_bl78bJLUQ"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KKhoowubJLUQ"
      },
      "source": [
        "## 12. Resolve API Base URL & Initialize Client\n",
        "We attempt to resolve the Moral Compass API base URL using:\n",
        "1. `MORAL_COMPASS_API_BASE_URL` environment variable.\n",
        "2. `get_api_base_url()` fallback.\n",
        "\n",
        "If resolution fails, the challenge section will be skipped."
      ],
      "id": "KKhoowubJLUQ"
    },
    {
      "cell_type": "code",
      "source": [
        "# manually setting this here, but needs to be dynamic in codebase\n",
        "os.environ['MORAL_COMPASS_API_BASE_URL'] = 'https://b22q73wp50.execute-api.us-east-1.amazonaws.com/dev'\n"
      ],
      "metadata": {
        "id": "BfOFoNHWMlWL"
      },
      "id": "BfOFoNHWMlWL",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-resolve-url",
        "outputId": "fb0b208c-6228-487f-9c41-08f34bca9f78",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Resolved Moral Compass API base URL: https://b22q73wp50.execute-api.us-east-1.amazonaws.com/dev\n"
          ]
        }
      ],
      "source": [
        "from aimodelshare.moral_compass import MoralcompassApiClient\n",
        "from aimodelshare.moral_compass.api_client import NotFoundError, ApiClientError\n",
        "from aimodelshare.moral_compass.challenge import ChallengeManager\n",
        "from aimodelshare.moral_compass.config import get_api_base_url\n",
        "\n",
        "def resolve_api_base_url():\n",
        "    env_url = os.getenv('MORAL_COMPASS_API_BASE_URL')\n",
        "    if env_url:\n",
        "        return env_url.rstrip('/')\n",
        "    try:\n",
        "        return get_api_base_url()\n",
        "    except RuntimeError as e:\n",
        "        raise RuntimeError(\n",
        "            \"Could not resolve API base URL. Set MORAL_COMPASS_API_BASE_URL or ensure terraform outputs are accessible.\"\n",
        "        ) from e\n",
        "\n",
        "try:\n",
        "    mc_api_base_url = resolve_api_base_url()\n",
        "    print(f\"Resolved Moral Compass API base URL: {mc_api_base_url}\")\n",
        "    mc_api_client = MoralcompassApiClient(api_base_url=mc_api_base_url)\n",
        "    mc_available = True\n",
        "except Exception as e:\n",
        "    print(f\"Moral Compass API not available: {e}. Skipping challenge section.\")\n",
        "    mc_available = False"
      ],
      "id": "mc-resolve-url"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQomZhcnJLUQ"
      },
      "source": [
        "## 13. Create / Ensure Challenge Table\n",
        "We create (idempotently) a new challenge table for a Justice & Equity themed challenge.\n",
        "Naming convention: `<playground_id>-mc` or fallback if playground ID unavailable."
      ],
      "id": "mQomZhcnJLUQ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-create-table",
        "outputId": "8b2dcd27-5fdc-4876-d034-7e777f602e46",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Attempting to create/ensure table: compas_playground_notebook-mc\n",
            "Table creation invoked (may already exist).\n",
            "Table metadata confirmed.\n"
          ]
        }
      ],
      "source": [
        "if mc_available:\n",
        "    USERNAME = os.getenv('username') or input(\"Enter username for Moral Compass challenge: \") or 'notebook_user_mc'\n",
        "    base_playground_id = playground_id or 'compas_playground_notebook'\n",
        "    TABLE_ID = f\"{base_playground_id}-mc\"\n",
        "    PLAYGROUND_URL =playground.playground_url if playground else None\n",
        "\n",
        "    print(f\"Attempting to create/ensure table: {TABLE_ID}\")\n",
        "    try:\n",
        "        mc_api_client.create_table(\n",
        "            TABLE_ID,\n",
        "            display_name='Justice & Equity Challenge (Notebook)',\n",
        "            playground_url=PLAYGROUND_URL\n",
        "        )\n",
        "        print(\"Table creation invoked (may already exist).\")\n",
        "    except Exception as e:\n",
        "        print(f\"Table creation skipped/failed (likely exists): {e}\")\n",
        "\n",
        "    # Confirm availability with retries\n",
        "    import time\n",
        "    max_retries = 8\n",
        "    for attempt in range(max_retries):\n",
        "        try:\n",
        "            mc_api_client.get_table(TABLE_ID)\n",
        "            print(\"Table metadata confirmed.\")\n",
        "            break\n",
        "        except (NotFoundError, ApiClientError) as e:\n",
        "            if attempt == max_retries - 1:\n",
        "                print(f\"Failed to confirm table after {max_retries} attempts: {e}\")\n",
        "                mc_available = False\n",
        "            else:\n",
        "                time.sleep(0.6)\n",
        "else:\n",
        "    print(\"Skipping table creation (API unavailable).\")"
      ],
      "id": "mc-create-table"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2XWfNpK8JLUQ"
      },
      "source": [
        "## 14. Pre-Sync Smoke Test\n",
        "Submit a minimalist update to ensure the endpoint accepts metrics and returns a `moralCompassScore`."
      ],
      "id": "2XWfNpK8JLUQ"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-smoke-test",
        "outputId": "5231aa9c-49d5-4549-dce1-057a1df90a65",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "✓ Smoke test passed. Response: {'username': 'mikedparrott', 'metrics': {'accuracy': 0.5}, 'primaryMetric': 'accuracy', 'moralCompassScore': 0.0, 'tasksCompleted': 0, 'totalTasks': 6, 'questionsCorrect': 0, 'totalQuestions': 14, 'message': 'Moral compass data updated successfully', 'createdNew': True}\n"
          ]
        }
      ],
      "source": [
        "if mc_available:\n",
        "    try:\n",
        "        smoke_resp = mc_api_client.update_moral_compass(\n",
        "            table_id=TABLE_ID,\n",
        "            username=USERNAME,\n",
        "            metrics={'accuracy': 0.5},\n",
        "            tasks_completed=0,\n",
        "            total_tasks=6,\n",
        "            questions_correct=0,\n",
        "            total_questions=14\n",
        "        )\n",
        "        assert 'moralCompassScore' in smoke_resp, 'Expected moralCompassScore in smoke response.'\n",
        "        print(f\"✓ Smoke test passed. Response: {smoke_resp}\")\n",
        "    except Exception as e:\n",
        "        print(f\"Smoke test failed: {e}\")\n",
        "        mc_available = False\n",
        "else:\n",
        "    print(\"Skipping smoke test (API unavailable).\")"
      ],
      "id": "mc-smoke-test"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yfeee6kKJLUR"
      },
      "source": [
        "## 15. Build Synthetic Dataset & Select Best Model\n",
        "We reproduce a small synthetic COMPAS-like dataset, train logistic regressions with different `C` values, and select the best accuracy for reporting to the challenge."
      ],
      "id": "Yfeee6kKJLUR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-build-dataset",
        "outputId": "76e86bd0-de67-4c72-8f24-7601b547fe36",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "C=0.1 | Accuracy=0.5600\n",
            "C=1.0 | Accuracy=0.5600\n",
            "C=3.0 | Accuracy=0.5600\n",
            "C=5 | Accuracy=0.5600\n",
            "Best C: 0.1 with accuracy=0.5600\n"
          ]
        }
      ],
      "source": [
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "def build_mc_dataset(n=200, seed=42):\n",
        "    rng = np.random.default_rng(seed)\n",
        "    race = rng.choice(['Black','White'], size=n, p=[0.5,0.5])\n",
        "    sex = rng.choice(['Male','Female'], size=n, p=[0.6,0.4])\n",
        "    age = rng.integers(18, 60, size=n)\n",
        "    priors = rng.integers(0, 15, size=n)\n",
        "    base = 0.3 + 0.03*priors + (race == 'Black')*0.05 + (sex == 'Male')*0.02\n",
        "    prob = 1/(1+np.exp(-(base - 0.5)))\n",
        "    label = (rng.random(n) < prob).astype(int)\n",
        "    df = pd.DataFrame({'race':race,'sex':sex,'age':age,'priors':priors,'label':label})\n",
        "    return df\n",
        "\n",
        "def featurize_mc(df):\n",
        "    d = df.copy()\n",
        "    d['race_Black'] = (d['race']=='Black').astype(int)\n",
        "    d['sex_Male'] = (d['sex']=='Male').astype(int)\n",
        "    X = d[['age','priors','race_Black','sex_Male']]\n",
        "    y = d['label']\n",
        "    return X, y\n",
        "\n",
        "def train_lr_variant(X, y, C):\n",
        "    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.25, random_state=13)\n",
        "    pipe = Pipeline([\n",
        "        ('scaler', StandardScaler()),\n",
        "        ('lr', LogisticRegression(C=C, max_iter=2000))\n",
        "    ])\n",
        "    pipe.fit(Xtr, ytr)\n",
        "    acc = pipe.score(Xte, yte)\n",
        "    return pipe, acc\n",
        "\n",
        "if mc_available:\n",
        "    df_mc = build_mc_dataset()\n",
        "    X_mc, y_mc = featurize_mc(df_mc)\n",
        "    candidate_Cs = [0.1, 1.0, 3.0,5]\n",
        "    best_acc = -1.0\n",
        "    best_C = None\n",
        "    for C in candidate_Cs:\n",
        "        modelC, accC = train_lr_variant(X_mc, y_mc, C)\n",
        "        print(f\"C={C} | Accuracy={accC:.4f}\")\n",
        "        if accC > best_acc:\n",
        "            best_acc = accC\n",
        "            best_C = C\n",
        "    print(f\"Best C: {best_C} with accuracy={best_acc:.4f}\")\n",
        "else:\n",
        "    print(\"Skipping model selection (API unavailable).\")"
      ],
      "id": "mc-build-dataset"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3rX10ns0JLUR"
      },
      "source": [
        "## 16. Initialize Challenge Manager & Set Metric\n",
        "We record the best accuracy as the primary metric for the user in the challenge."
      ],
      "id": "3rX10ns0JLUR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-init-manager",
        "outputId": "65c98e00-a655-493a-b7c5-8956b3bfadde",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Primary metric 'accuracy' set to 0.5600\n"
          ]
        }
      ],
      "source": [
        "from aimodelshare.moral_compass.challenge import ChallengeManager\n",
        "\n",
        "if mc_available:\n",
        "    try:\n",
        "        manager = ChallengeManager(table_id=TABLE_ID, username=USERNAME, api_client=mc_api_client)\n",
        "        manager.set_metric('accuracy', best_acc, primary=True)\n",
        "        print(f\"Primary metric 'accuracy' set to {best_acc:.4f}\")\n",
        "    except Exception as e:\n",
        "        print(f\"Failed to initialize ChallengeManager: {e}\")\n",
        "        mc_available = False\n",
        "else:\n",
        "    print(\"Skipping manager initialization (API unavailable).\")"
      ],
      "id": "mc-init-manager"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NbyIjn7PJLUR"
      },
      "source": [
        "## 17. Progress Through Tasks & Questions\n",
        "We iterate through all tasks, completing them and answering each question with the correct index. After each task block, we sync with the server and monitor score progression."
      ],
      "id": "NbyIjn7PJLUR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-progress",
        "outputId": "9c47752d-c6d9-43c5-86cf-6c7f78ce8045",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Task: A - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.0933\n",
            "\n",
            "Task: B - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.1867\n",
            "\n",
            "Task: C - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.2800\n",
            "\n",
            "Task: D - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.3733\n",
            "\n",
            "Task: E - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.4667\n",
            "\n",
            "Task: F - Completing and answering questions...\n",
            "Synced. Moral Compass Score: 0.5600\n",
            "\n",
            "All tasks completed and synced.\n"
          ]
        }
      ],
      "source": [
        "if mc_available:\n",
        "    try:\n",
        "        challenge = manager.challenge\n",
        "        prev_score = 0.0\n",
        "        for task in challenge.tasks:\n",
        "            print(f\"\\nTask: {task.id} - Completing and answering questions...\")\n",
        "            manager.complete_task(task.id)\n",
        "            for q in task.questions:\n",
        "                manager.answer_question(task.id, q.id, selected_index=q.correct_index)\n",
        "            sync_resp = manager.sync()\n",
        "            mc_score = sync_resp.get('moralCompassScore', 0)\n",
        "            print(f\"Synced. Moral Compass Score: {mc_score:.4f}\")\n",
        "            if mc_score + 1e-9 < prev_score:\n",
        "                print(f\"Warning: Score decreased from {prev_score:.4f} to {mc_score:.4f}\")\n",
        "            prev_score = mc_score\n",
        "        print(\"\\nAll tasks completed and synced.\")\n",
        "    except Exception as e:\n",
        "        print(f\"Error during task progression: {e}\")\n",
        "        mc_available = False\n",
        "else:\n",
        "    print(\"Skipping task progression (API unavailable).\")"
      ],
      "id": "mc-progress"
    },
    {
      "cell_type": "code",
      "source": [
        "# Increase accuracy after moral compass score maxed out\n",
        "manager.set_metric('accuracy', .77, primary=True)\n",
        "sync_resp = manager.sync()"
      ],
      "metadata": {
        "id": "zBTyu3iIPKIt"
      },
      "id": "zBTyu3iIPKIt",
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nTp2Swj4JLUR"
      },
      "source": [
        "## 18. Final Summary & Leaderboard Validation\n",
        "We retrieve the user's progress summary and verify leaderboard entry alignment with local score preview."
      ],
      "id": "nTp2Swj4JLUR"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mc-final-summary",
        "outputId": "3ede9e1e-5932-4e59-8878-b457a0941ea3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Progress Summary:\n",
            "  tasksCompleted: 6\n",
            "  totalTasks: 6\n",
            "  questionsCorrect: 6\n",
            "  totalQuestions: 6\n",
            "  metrics: {'accuracy': 0.77}\n",
            "  primaryMetric: accuracy\n",
            "  localScorePreview: 0.77\n",
            "\n",
            "Leaderboard Entry:\n",
            "  username: mikedparrott\n",
            "  submissionCount: 0\n",
            "  totalCount: 0\n",
            "  lastUpdated: 2025-10-29T09:14:07.776905\n",
            "  moralCompassScore: 0.77\n",
            "  metrics: {'accuracy': 0.77}\n",
            "  primaryMetric: accuracy\n",
            "  tasksCompleted: 6\n",
            "  totalTasks: 6\n",
            "  questionsCorrect: 6\n",
            "  totalQuestions: 6\n",
            "✓ Leaderboard score aligned sufficiently with local preview.\n"
          ]
        }
      ],
      "source": [
        "if mc_available:\n",
        "    try:\n",
        "        summary = manager.get_progress_summary()\n",
        "        print(\"Progress Summary:\")\n",
        "        for k, v in summary.items():\n",
        "            print(f\"  {k}: {v}\")\n",
        "        assert summary['tasksCompleted'] == summary['totalTasks'], 'Not all tasks completed.'\n",
        "        assert summary['questionsCorrect'] == summary['totalQuestions'], 'Not all questions answered correctly.'\n",
        "        final_local = summary.get('localScorePreview', 0)\n",
        "        assert final_local > 0, 'Final local score should be positive.'\n",
        "\n",
        "        lb = mc_api_client.list_users(TABLE_ID, limit=100)\n",
        "        entries = [u for u in lb.get('users', []) if u.get('username') == USERNAME]\n",
        "        if not entries:\n",
        "            print(\"User not found on leaderboard.\")\n",
        "        else:\n",
        "            user_entry = entries[0]\n",
        "            print(\"\\nLeaderboard Entry:\")\n",
        "            for k, v in user_entry.items():\n",
        "                print(f\"  {k}: {v}\")\n",
        "            mc_score_lb = user_entry.get('moralCompassScore', 0)\n",
        "            if mc_score_lb + 0.2 < final_local:\n",
        "                print(\"Leaderboard score appears misaligned with local preview.\")\n",
        "            else:\n",
        "                print(\"✓ Leaderboard score aligned sufficiently with local preview.\")\n",
        "    except Exception as e:\n",
        "        print(f\"Final summary/leaderboard validation failed: {e}\")\n",
        "else:\n",
        "    print(\"Skipping final summary (API unavailable).\")"
      ],
      "id": "mc-final-summary"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hmaFnMQcJLUR"
      },
      "source": [
        "## 19. Challenge Section Complete\n",
        "You have (if API was available) created a challenge table, submitted metric progress, completed tasks & questions, and validated the leaderboard entry.\n",
        "\n",
        "Feel free to modify:\n",
        "- Hyperparameter candidates\n",
        "- Additional metrics\n",
        "- Custom scoring logic\n",
        "- Partial task completion for experimental flows\n",
        "\n",
        "End of notebook."
      ],
      "id": "hmaFnMQcJLUR"
    }
  ]
}