{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Tutorial 4: Working with Geometries\n",
                "\n",
                "cendat can fetch geographic boundaries alongside your data for mapping.\n",
                "\n",
                "**Goal:** Get median household income data with geometries for Colorado block groups and create a choropleth map."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Setup"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "✅ API key loaded successfully.\n"
                    ]
                }
            ],
            "source": [
                "import os\n",
                "import matplotlib.pyplot as plt\n",
                "from cendat import CenDatHelper\n",
                "from dotenv import load_dotenv\n",
                "\n",
                "load_dotenv()\n",
                "cdh = CenDatHelper(key=os.getenv(\"CENSUS_API_KEY\"))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Step 1: Select Product"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "✅ Product set: 'ACS 5-Year Detailed Tables (2023/acs/acs5)' (Vintage: [2023])\n"
                    ]
                }
            ],
            "source": [
                "cdh.list_products(years=[2023], patterns=r\"acs/acs5\\)\")\n",
                "cdh.set_products()"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Step 2: Select Variables and Geography"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "[{'name': 'B29004',\n",
                            "  'description': 'Median Household Income for Households With a Citizen, Voting-Age Householder (in 2023 Inflation-Adjusted Dollars)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013H',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (White Alone, Not Hispanic or Latino Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013I',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Hispanic or Latino Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013B',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Black or African American Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013C',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (American Indian and Alaska Native Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013A',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (White Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013F',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Some Other Race Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013G',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Two or More Races Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013D',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Asian Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013E',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) (Native Hawaiian and Other Pacific Islander Alone Householder)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B25119',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) by Tenure',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19019',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) by Household Size',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19013',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars)',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B22008',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) by Receipt of Food Stamps/SNAP in the Past 12 Months',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'},\n",
                            " {'name': 'B19049',\n",
                            "  'description': 'Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars) by Age of Householder',\n",
                            "  'product': 'ACS 5-Year Detailed Tables (2023/acs/acs5)',\n",
                            "  'vintage': [2023],\n",
                            "  'url': 'http://api.census.gov/data/2023/acs/acs5'}]"
                        ]
                    },
                    "execution_count": 10,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "# Search for median household income groups\n",
                "cdh.list_groups(patterns=r\"^median household income\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "✅ Groups set: B19013\n",
                        "\n",
                        "--- Group: B19013 (Median Household Income in the Past 12 Months (in 2023 Inflation-Adjusted Dollars)) ---\n",
                        "\n",
                        "  Product: ACS 5-Year Detailed Tables (2023/acs/acs5) (Vintage: 2023)\n",
                        "      B19013_001E: Median household income in the past 12 months (in 2023 inflation-adjusted dollars)\n"
                    ]
                }
            ],
            "source": [
                "cdh.set_groups([\"B19013\"])\n",
                "cdh.describe_groups()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "✅ Geographies set: 'block group' (requires `within` for: county, state, tract)\n"
                    ]
                }
            ],
            "source": [
                "# 150 = Block Groups\n",
                "cdh.set_geos([\"150\"])"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Step 3: Get Data with Geometry"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "✅ Variables set:\n",
                        "  - Product: ACS 5-Year Detailed Tables (2023/acs/acs5) (Vintage: [2023])\n",
                        "    Variables: B19013_001E\n",
                        "✅ Parameters created for 1 geo-variable/group combinations.\n",
                        "✅ Data fetching complete. Stacking results.\n",
                        "✅ Geometry fetching complete. Stacking results.\n"
                    ]
                }
            ],
            "source": [
                "# Fetch data for 3 Colorado counties with geometries\n",
                "response = cdh.get_data(\n",
                "    include_names=True,\n",
                "    include_geometry=True,  # This fetches boundaries from TIGERweb\n",
                "    within={\n",
                "        \"state\": [\"08\"],  # Colorado\n",
                "        \"county\": [\"069\", \"123\", \"013\"],  # Larimer, Weld, Boulder\n",
                "    },\n",
                ")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Step 4: Convert to GeoDataFrame and Map"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Block groups returned: 669\n"
                    ]
                },
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>NAME</th>\n",
                            "      <th>GEO_ID</th>\n",
                            "      <th>B19013_001E</th>\n",
                            "      <th>B19013_001EA</th>\n",
                            "      <th>B19013_001M</th>\n",
                            "      <th>B19013_001MA</th>\n",
                            "      <th>state</th>\n",
                            "      <th>county</th>\n",
                            "      <th>tract</th>\n",
                            "      <th>block group</th>\n",
                            "      <th>GEOID</th>\n",
                            "      <th>geometry</th>\n",
                            "      <th>product</th>\n",
                            "      <th>vintage</th>\n",
                            "      <th>sumlev</th>\n",
                            "      <th>desc</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>Block Group 1; Census Tract 19.05; Weld County...</td>\n",
                            "      <td>1500000US081230019051</td>\n",
                            "      <td>114333.0</td>\n",
                            "      <td>None</td>\n",
                            "      <td>23010</td>\n",
                            "      <td>None</td>\n",
                            "      <td>08</td>\n",
                            "      <td>123</td>\n",
                            "      <td>001905</td>\n",
                            "      <td>1</td>\n",
                            "      <td>081230019051</td>\n",
                            "      <td>POLYGON ((-104.80157 40.08309, -104.8012 40.08...</td>\n",
                            "      <td>ACS 5-Year Detailed Tables (2023/acs/acs5)</td>\n",
                            "      <td>2023</td>\n",
                            "      <td>150</td>\n",
                            "      <td>block group</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Block Group 2; Census Tract 19.05; Weld County...</td>\n",
                            "      <td>1500000US081230019052</td>\n",
                            "      <td>34250.0</td>\n",
                            "      <td>None</td>\n",
                            "      <td>12371</td>\n",
                            "      <td>None</td>\n",
                            "      <td>08</td>\n",
                            "      <td>123</td>\n",
                            "      <td>001905</td>\n",
                            "      <td>2</td>\n",
                            "      <td>081230019052</td>\n",
                            "      <td>POLYGON ((-104.81053 40.08671, -104.81053 40.0...</td>\n",
                            "      <td>ACS 5-Year Detailed Tables (2023/acs/acs5)</td>\n",
                            "      <td>2023</td>\n",
                            "      <td>150</td>\n",
                            "      <td>block group</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>Block Group 3; Census Tract 19.05; Weld County...</td>\n",
                            "      <td>1500000US081230019053</td>\n",
                            "      <td>74052.0</td>\n",
                            "      <td>None</td>\n",
                            "      <td>30102</td>\n",
                            "      <td>None</td>\n",
                            "      <td>08</td>\n",
                            "      <td>123</td>\n",
                            "      <td>001905</td>\n",
                            "      <td>3</td>\n",
                            "      <td>081230019053</td>\n",
                            "      <td>POLYGON ((-104.81055 40.09454, -104.81055 40.0...</td>\n",
                            "      <td>ACS 5-Year Detailed Tables (2023/acs/acs5)</td>\n",
                            "      <td>2023</td>\n",
                            "      <td>150</td>\n",
                            "      <td>block group</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>Block Group 1; Census Tract 19.06; Weld County...</td>\n",
                            "      <td>1500000US081230019061</td>\n",
                            "      <td>74063.0</td>\n",
                            "      <td>None</td>\n",
                            "      <td>12646</td>\n",
                            "      <td>None</td>\n",
                            "      <td>08</td>\n",
                            "      <td>123</td>\n",
                            "      <td>001906</td>\n",
                            "      <td>1</td>\n",
                            "      <td>081230019061</td>\n",
                            "      <td>POLYGON ((-104.8862 40.12156, -104.8862 40.121...</td>\n",
                            "      <td>ACS 5-Year Detailed Tables (2023/acs/acs5)</td>\n",
                            "      <td>2023</td>\n",
                            "      <td>150</td>\n",
                            "      <td>block group</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>Block Group 1; Census Tract 20.18; Weld County...</td>\n",
                            "      <td>1500000US081230020181</td>\n",
                            "      <td>142338.0</td>\n",
                            "      <td>None</td>\n",
                            "      <td>14844</td>\n",
                            "      <td>None</td>\n",
                            "      <td>08</td>\n",
                            "      <td>123</td>\n",
                            "      <td>002018</td>\n",
                            "      <td>1</td>\n",
                            "      <td>081230020181</td>\n",
                            "      <td>POLYGON ((-104.98033 40.13875, -104.98033 40.1...</td>\n",
                            "      <td>ACS 5-Year Detailed Tables (2023/acs/acs5)</td>\n",
                            "      <td>2023</td>\n",
                            "      <td>150</td>\n",
                            "      <td>block group</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                                NAME                 GEO_ID  \\\n",
                            "0  Block Group 1; Census Tract 19.05; Weld County...  1500000US081230019051   \n",
                            "1  Block Group 2; Census Tract 19.05; Weld County...  1500000US081230019052   \n",
                            "2  Block Group 3; Census Tract 19.05; Weld County...  1500000US081230019053   \n",
                            "3  Block Group 1; Census Tract 19.06; Weld County...  1500000US081230019061   \n",
                            "4  Block Group 1; Census Tract 20.18; Weld County...  1500000US081230020181   \n",
                            "\n",
                            "   B19013_001E B19013_001EA  B19013_001M B19013_001MA state county   tract  \\\n",
                            "0     114333.0         None        23010         None    08    123  001905   \n",
                            "1      34250.0         None        12371         None    08    123  001905   \n",
                            "2      74052.0         None        30102         None    08    123  001905   \n",
                            "3      74063.0         None        12646         None    08    123  001906   \n",
                            "4     142338.0         None        14844         None    08    123  002018   \n",
                            "\n",
                            "  block group         GEOID  \\\n",
                            "0           1  081230019051   \n",
                            "1           2  081230019052   \n",
                            "2           3  081230019053   \n",
                            "3           1  081230019061   \n",
                            "4           1  081230020181   \n",
                            "\n",
                            "                                            geometry  \\\n",
                            "0  POLYGON ((-104.80157 40.08309, -104.8012 40.08...   \n",
                            "1  POLYGON ((-104.81053 40.08671, -104.81053 40.0...   \n",
                            "2  POLYGON ((-104.81055 40.09454, -104.81055 40.0...   \n",
                            "3  POLYGON ((-104.8862 40.12156, -104.8862 40.121...   \n",
                            "4  POLYGON ((-104.98033 40.13875, -104.98033 40.1...   \n",
                            "\n",
                            "                                      product vintage sumlev         desc  \n",
                            "0  ACS 5-Year Detailed Tables (2023/acs/acs5)    2023    150  block group  \n",
                            "1  ACS 5-Year Detailed Tables (2023/acs/acs5)    2023    150  block group  \n",
                            "2  ACS 5-Year Detailed Tables (2023/acs/acs5)    2023    150  block group  \n",
                            "3  ACS 5-Year Detailed Tables (2023/acs/acs5)    2023    150  block group  \n",
                            "4  ACS 5-Year Detailed Tables (2023/acs/acs5)    2023    150  block group  "
                        ]
                    },
                    "execution_count": 14,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "# to_gpd() returns a GeoDataFrame ready for mapping\n",
                "gdf = response.to_gpd(destring=True, join_strategy=\"inner\")\n",
                "\n",
                "# Handle missing values (Census uses -666666666 for suppressed data)\n",
                "gdf.loc[gdf[\"B19013_001E\"] == -666666666, \"B19013_001E\"] = None\n",
                "\n",
                "print(f\"Block groups returned: {len(gdf)}\")\n",
                "gdf.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "image/png": "[encoded data, 94848 characters]",
                        "text/plain": [
                            "<Figure size 1000x500 with 2 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# Create a polished choropleth map\n",
                "fig, ax = plt.subplots(1, 1, figsize=(10, 5))\n",
                "\n",
                "gdf.plot(\n",
                "    column=\"B19013_001E\",\n",
                "    cmap=\"viridis\",\n",
                "    linewidth=0.8,\n",
                "    ax=ax,\n",
                "    legend=True,\n",
                "    alpha=1.0,\n",
                "    legend_kwds={\n",
                "        \"label\": \"Income\",\n",
                "        \"orientation\": \"horizontal\",\n",
                "        \"location\": \"bottom\",\n",
                "        \"shrink\": 0.5,\n",
                "        \"fraction\": 0.1,\n",
                "        \"format\": \"{x:,.0f}\",\n",
                "    },\n",
                "    missing_kwds={\n",
                "        \"color\": \"lightgrey\",\n",
                "        \"edgecolor\": \"grey\",\n",
                "        \"hatch\": \"////\",\n",
                "        \"label\": \"Missing values\",\n",
                "    },\n",
                ")\n",
                "\n",
                "ax.set_title(\n",
                "    \"Larimer, Weld, and Boulder County Med. HH Income by Block Group\",\n",
                "    fontdict={\"fontsize\": \"12\", \"fontweight\": \"3\"},\n",
                ")\n",
                "ax.set_axis_off()\n",
                "plt.tight_layout()\n",
                "plt.show()"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "> **Note - Supported Geographies**\n",
                ">\n",
                "> Geometry fetching currently supports: regions (020), divisions (030), states (040), counties (050), county subdivisions (060), census tracts (140), block groups (150), and places (160)."
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "name": "python",
            "version": "3.11.0"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 4
}
