{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n",
      "Executing command: C:\\ProgramData\\Anaconda3\\python.exe -m pip install -U pandasql\n",
      "b\"Collecting pandasql\\r\\n  Downloading pandasql-0.7.3.tar.gz (26 kB)\\r\\nRequirement already satisfied: numpy in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from pandasql) (1.22.2)\\r\\nRequirement already satisfied: pandas in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from pandasql) (1.3.4)\\r\\nRequirement already satisfied: sqlalchemy in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from pandasql) (1.4.22)\\r\\nRequirement already satisfied: python-dateutil>=2.7.3 in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from pandas->pandasql) (2.8.2)\\r\\nRequirement already satisfied: pytz>=2017.3 in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from pandas->pandasql) (2021.3)\\r\\nRequirement already satisfied: six>=1.5 in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from python-dateutil>=2.7.3->pandas->pandasql) (1.16.0)\\r\\nRequirement already satisfied: greenlet!=0.4.17 in c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages (from sqlalchemy->pandasql) (1.1.1)\\r\\nBuilding wheels for collected packages: pandasql\\r\\n  Building wheel for pandasql (setup.py): started\\r\\n  Building wheel for pandasql (setup.py): finished with status 'done'\\r\\n  Created wheel for pandasql: filename=pandasql-0.7.3-py3-none-any.whl size=26781 sha256=ee1d8d0ef4f2452d0b2aad7e4bf722db4173906b5ea49e78a36c310cea3643df\\r\\n  Stored in directory: c:\\\\users\\\\ahmet\\\\appdata\\\\local\\\\pip\\\\cache\\\\wheels\\\\63\\\\e8\\\\ec\\\\75b1df467ecf57b6ececb32cb16f4e86697cbfe55cb0c51f07\\r\\nSuccessfully built pandasql\\r\\nInstalling collected packages: pandasql\\r\\nSuccessfully installed pandasql-0.7.3\\r\\n\"\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys,os; sys.path.append(os.environ['BMESAHMETDIR']); import bmes\n",
    "bmes.pipinstall('Bio','biopython')\n",
    "bmes.pipinstall('GEOparse')\n",
    "bmes.pipinstall('pandasql')\n",
    "\n",
    "\n",
    "import GEOparse\n",
    "import pandas as pd\n",
    "import numpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and parse GEO data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12-Apr-2022 15:10:46 DEBUG utils - Directory C:/Users/ahmet/AppData/Local/Temp/bmes already exists. Skipping.\n",
      "12-Apr-2022 15:10:46 INFO GEOparse - File already exist: using local version.\n",
      "12-Apr-2022 15:10:46 INFO GEOparse - Parsing C:/Users/ahmet/AppData/Local/Temp/bmes\\GSE5847_family.soft.gz: \n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - DATABASE: GeoMiame\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SERIES: GSE5847\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - PLATFORM: GPL96\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136326\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136327\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136328\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136329\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136330\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136331\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136332\n",
      "12-Apr-2022 15:10:46 DEBUG GEOparse - SAMPLE: GSM136333\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136334\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136335\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136336\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136337\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136338\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136339\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136340\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136341\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136342\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136343\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136344\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136345\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136346\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136347\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136348\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136349\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136350\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136351\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136352\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136353\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136354\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136355\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136356\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136357\n",
      "12-Apr-2022 15:10:47 DEBUG GEOparse - SAMPLE: GSM136358\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136359\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136360\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136361\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136362\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136363\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136364\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136365\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136366\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136367\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136368\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136369\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136370\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136371\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136372\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136373\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136374\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136375\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136376\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136377\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136378\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136379\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136380\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136381\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136382\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136383\n",
      "12-Apr-2022 15:10:48 DEBUG GEOparse - SAMPLE: GSM136384\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136385\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136386\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136387\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136388\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136389\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136390\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136391\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136392\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136393\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136394\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136395\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136396\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136397\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136398\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136399\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136400\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136401\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136402\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136403\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136404\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136405\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136406\n",
      "12-Apr-2022 15:10:49 DEBUG GEOparse - SAMPLE: GSM136407\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136408\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136409\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136410\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136411\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136412\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136413\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136414\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136415\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136416\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136417\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136418\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136419\n",
      "12-Apr-2022 15:10:50 DEBUG GEOparse - SAMPLE: GSM136420\n"
     ]
    }
   ],
   "source": [
    "# TODO: this will download data everytime this section is executed. Can you optimize?\n",
    "gse = GEOparse.get_GEO(geo=\"GSE5847\", destdir=bmes.tempdir());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'title': ['LCM stroma sample from patient #37'],\n",
       " 'geo_accession': ['GSM136326'],\n",
       " 'status': ['Public on Sep 30 2007'],\n",
       " 'submission_date': ['Sep 15 2006'],\n",
       " 'last_update_date': ['Feb 17 2016'],\n",
       " 'type': ['RNA'],\n",
       " 'channel_count': ['1'],\n",
       " 'source_name_ch1': ['human breast cancer stroma'],\n",
       " 'organism_ch1': ['Homo sapiens'],\n",
       " 'taxid_ch1': ['9606'],\n",
       " 'characteristics_ch1': ['diagnosis: IBC', 'status: Deceased'],\n",
       " 'molecule_ch1': ['total RNA'],\n",
       " 'extract_protocol_ch1': ['PicoPure kit (Arcturus)'],\n",
       " 'label_ch1': ['biotin'],\n",
       " 'label_protocol_ch1': ['Small Sample Labeling Protocol vII from Affymetrix'],\n",
       " 'hyb_protocol': ['Standard Affy protocol'],\n",
       " 'scan_protocol': ['Standard Affy protocol'],\n",
       " 'description': ['Chemo: Yes, ER: NEG, Her2Neu: NEG, Stage: IIIA, Clinical IBC: No'],\n",
       " 'data_processing': ['RMA'],\n",
       " 'platform_id': ['GPL96'],\n",
       " 'contact_name': ['Stefan,,Ambs'],\n",
       " 'contact_laboratory': ['LHC'],\n",
       " 'contact_institute': ['NCI'],\n",
       " 'contact_address': ['37 Convent Dr Bldg 37 Room 3050'],\n",
       " 'contact_city': ['Bethesda'],\n",
       " 'contact_state': ['MD'],\n",
       " 'contact_zip/postal_code': ['20892'],\n",
       " 'contact_country': ['USA'],\n",
       " 'supplementary_file': ['ftp://ftp.ncbi.nlm.nih.gov/geo/samples/GSM136nnn/GSM136326/suppl/GSM136326.CEL.gz'],\n",
       " 'relation': ['Reanalyzed by: GSE47561'],\n",
       " 'series_id': ['GSE5847'],\n",
       " 'data_row_count': ['22283']}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Each GSM sample is available in gse.gsms[gsmid]\n",
    "# Information about each sample is available in gse.gsms[...].metadata\n",
    "gsm= list(gse.gsms.values())[0]\n",
    "gsm.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GSM136326</th>\n",
       "      <th>GSM136327</th>\n",
       "      <th>GSM136328</th>\n",
       "      <th>GSM136329</th>\n",
       "      <th>GSM136330</th>\n",
       "      <th>GSM136331</th>\n",
       "      <th>GSM136332</th>\n",
       "      <th>GSM136333</th>\n",
       "      <th>GSM136334</th>\n",
       "      <th>GSM136335</th>\n",
       "      <th>...</th>\n",
       "      <th>GSM136411</th>\n",
       "      <th>GSM136412</th>\n",
       "      <th>GSM136413</th>\n",
       "      <th>GSM136414</th>\n",
       "      <th>GSM136415</th>\n",
       "      <th>GSM136416</th>\n",
       "      <th>GSM136417</th>\n",
       "      <th>GSM136418</th>\n",
       "      <th>GSM136419</th>\n",
       "      <th>GSM136420</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_REF</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1007_s_at</th>\n",
       "      <td>10.450276</td>\n",
       "      <td>9.399542</td>\n",
       "      <td>9.424799</td>\n",
       "      <td>9.472922</td>\n",
       "      <td>9.278780</td>\n",
       "      <td>9.434428</td>\n",
       "      <td>9.924859</td>\n",
       "      <td>9.561787</td>\n",
       "      <td>9.745115</td>\n",
       "      <td>9.508479</td>\n",
       "      <td>...</td>\n",
       "      <td>10.679646</td>\n",
       "      <td>9.804469</td>\n",
       "      <td>10.562347</td>\n",
       "      <td>10.792802</td>\n",
       "      <td>10.250198</td>\n",
       "      <td>10.762713</td>\n",
       "      <td>10.560312</td>\n",
       "      <td>10.757617</td>\n",
       "      <td>10.888868</td>\n",
       "      <td>10.772049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053_at</th>\n",
       "      <td>5.719466</td>\n",
       "      <td>4.849293</td>\n",
       "      <td>4.732081</td>\n",
       "      <td>4.728854</td>\n",
       "      <td>5.326392</td>\n",
       "      <td>5.230320</td>\n",
       "      <td>5.215126</td>\n",
       "      <td>4.834023</td>\n",
       "      <td>4.877629</td>\n",
       "      <td>5.033619</td>\n",
       "      <td>...</td>\n",
       "      <td>5.684671</td>\n",
       "      <td>5.726441</td>\n",
       "      <td>5.731007</td>\n",
       "      <td>5.646109</td>\n",
       "      <td>5.690716</td>\n",
       "      <td>5.873947</td>\n",
       "      <td>5.630389</td>\n",
       "      <td>5.527384</td>\n",
       "      <td>6.216526</td>\n",
       "      <td>6.076195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117_at</th>\n",
       "      <td>5.938664</td>\n",
       "      <td>6.083273</td>\n",
       "      <td>6.447978</td>\n",
       "      <td>6.176949</td>\n",
       "      <td>6.544585</td>\n",
       "      <td>6.077795</td>\n",
       "      <td>6.293336</td>\n",
       "      <td>6.206549</td>\n",
       "      <td>5.916959</td>\n",
       "      <td>6.633561</td>\n",
       "      <td>...</td>\n",
       "      <td>6.418576</td>\n",
       "      <td>6.725075</td>\n",
       "      <td>6.420450</td>\n",
       "      <td>6.504424</td>\n",
       "      <td>6.602684</td>\n",
       "      <td>6.698563</td>\n",
       "      <td>6.424410</td>\n",
       "      <td>6.426399</td>\n",
       "      <td>6.494861</td>\n",
       "      <td>6.258779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121_at</th>\n",
       "      <td>8.023052</td>\n",
       "      <td>7.894659</td>\n",
       "      <td>8.344988</td>\n",
       "      <td>8.163204</td>\n",
       "      <td>8.233756</td>\n",
       "      <td>7.595106</td>\n",
       "      <td>7.916302</td>\n",
       "      <td>7.927066</td>\n",
       "      <td>8.032575</td>\n",
       "      <td>7.908482</td>\n",
       "      <td>...</td>\n",
       "      <td>8.253583</td>\n",
       "      <td>8.475521</td>\n",
       "      <td>8.177014</td>\n",
       "      <td>8.327302</td>\n",
       "      <td>8.275579</td>\n",
       "      <td>8.378434</td>\n",
       "      <td>8.430819</td>\n",
       "      <td>8.403584</td>\n",
       "      <td>8.691069</td>\n",
       "      <td>8.250358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1255_g_at</th>\n",
       "      <td>3.954803</td>\n",
       "      <td>3.963246</td>\n",
       "      <td>3.964102</td>\n",
       "      <td>4.087836</td>\n",
       "      <td>3.998893</td>\n",
       "      <td>3.839705</td>\n",
       "      <td>4.058756</td>\n",
       "      <td>3.958660</td>\n",
       "      <td>4.118736</td>\n",
       "      <td>4.025031</td>\n",
       "      <td>...</td>\n",
       "      <td>4.143628</td>\n",
       "      <td>4.231317</td>\n",
       "      <td>4.117592</td>\n",
       "      <td>4.229090</td>\n",
       "      <td>4.182036</td>\n",
       "      <td>4.102979</td>\n",
       "      <td>4.178124</td>\n",
       "      <td>4.353376</td>\n",
       "      <td>4.290219</td>\n",
       "      <td>4.107710</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           GSM136326  GSM136327  GSM136328  GSM136329  GSM136330  GSM136331  \\\n",
       "ID_REF                                                                        \n",
       "1007_s_at  10.450276   9.399542   9.424799   9.472922   9.278780   9.434428   \n",
       "1053_at     5.719466   4.849293   4.732081   4.728854   5.326392   5.230320   \n",
       "117_at      5.938664   6.083273   6.447978   6.176949   6.544585   6.077795   \n",
       "121_at      8.023052   7.894659   8.344988   8.163204   8.233756   7.595106   \n",
       "1255_g_at   3.954803   3.963246   3.964102   4.087836   3.998893   3.839705   \n",
       "\n",
       "           GSM136332  GSM136333  GSM136334  GSM136335  ...  GSM136411  \\\n",
       "ID_REF                                                 ...              \n",
       "1007_s_at   9.924859   9.561787   9.745115   9.508479  ...  10.679646   \n",
       "1053_at     5.215126   4.834023   4.877629   5.033619  ...   5.684671   \n",
       "117_at      6.293336   6.206549   5.916959   6.633561  ...   6.418576   \n",
       "121_at      7.916302   7.927066   8.032575   7.908482  ...   8.253583   \n",
       "1255_g_at   4.058756   3.958660   4.118736   4.025031  ...   4.143628   \n",
       "\n",
       "           GSM136412  GSM136413  GSM136414  GSM136415  GSM136416  GSM136417  \\\n",
       "ID_REF                                                                        \n",
       "1007_s_at   9.804469  10.562347  10.792802  10.250198  10.762713  10.560312   \n",
       "1053_at     5.726441   5.731007   5.646109   5.690716   5.873947   5.630389   \n",
       "117_at      6.725075   6.420450   6.504424   6.602684   6.698563   6.424410   \n",
       "121_at      8.475521   8.177014   8.327302   8.275579   8.378434   8.430819   \n",
       "1255_g_at   4.231317   4.117592   4.229090   4.182036   4.102979   4.178124   \n",
       "\n",
       "           GSM136418  GSM136419  GSM136420  \n",
       "ID_REF                                      \n",
       "1007_s_at  10.757617  10.888868  10.772049  \n",
       "1053_at     5.527384   6.216526   6.076195  \n",
       "117_at      6.426399   6.494861   6.258779  \n",
       "121_at      8.403584   8.691069   8.250358  \n",
       "1255_g_at   4.353376   4.290219   4.107710  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Data for each GSM is available in gse.gsms[...].table\n",
    "\n",
    "# Combine GSM's into a single data table.\n",
    "gsedata = None\n",
    "for gsmid in gse.gsms.keys():\n",
    "    gsmdata = gse.gsms[gsmid].table.rename(columns={'VALUE':gsmid});\n",
    "    if gsedata is None: gsedata=gsmdata;\n",
    "    else:\n",
    "        assert(gsedata['ID_REF'].equals(gsmdata['ID_REF'])) #just make sure that the same probes are listed in the same order.\n",
    "        gsedata = pd.concat([gsedata,gsmdata[gsmid]],axis=1);\n",
    "\n",
    "gsedata = gsedata.set_index('ID_REF')\n",
    "gsedata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<d: GPL96>\n"
     ]
    }
   ],
   "source": [
    "gplid = list(gse.gpls.values())[0]\n",
    "print(gplid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene Symbol</th>\n",
       "      <th>Gene Ontology Biological Process</th>\n",
       "      <th>Gene Ontology Cellular Component</th>\n",
       "      <th>Gene Ontology Molecular Function</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1007_s_at</td>\n",
       "      <td>DDR1 /// MIR4640</td>\n",
       "      <td>0001558 // regulation of cell growth // inferr...</td>\n",
       "      <td>0005576 // extracellular region // inferred fr...</td>\n",
       "      <td>0000166 // nucleotide binding // inferred from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1053_at</td>\n",
       "      <td>RFC2</td>\n",
       "      <td>0000278 // mitotic cell cycle // traceable aut...</td>\n",
       "      <td>0005634 // nucleus // inferred from electronic...</td>\n",
       "      <td>0000166 // nucleotide binding // inferred from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>117_at</td>\n",
       "      <td>HSPA6</td>\n",
       "      <td>0000902 // cell morphogenesis // inferred from...</td>\n",
       "      <td>0005737 // cytoplasm // inferred from direct a...</td>\n",
       "      <td>0000166 // nucleotide binding // inferred from...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>121_at</td>\n",
       "      <td>PAX8</td>\n",
       "      <td>0001655 // urogenital system development // in...</td>\n",
       "      <td>0005634 // nucleus // inferred from direct ass...</td>\n",
       "      <td>0000979 // RNA polymerase II core promoter seq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1255_g_at</td>\n",
       "      <td>GUCA1A</td>\n",
       "      <td>0007165 // signal transduction // non-traceabl...</td>\n",
       "      <td>0001750 // photoreceptor outer segment // infe...</td>\n",
       "      <td>0005509 // calcium ion binding // inferred fro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ID       Gene Symbol  \\\n",
       "0  1007_s_at  DDR1 /// MIR4640   \n",
       "1    1053_at              RFC2   \n",
       "2     117_at             HSPA6   \n",
       "3     121_at              PAX8   \n",
       "4  1255_g_at            GUCA1A   \n",
       "\n",
       "                    Gene Ontology Biological Process  \\\n",
       "0  0001558 // regulation of cell growth // inferr...   \n",
       "1  0000278 // mitotic cell cycle // traceable aut...   \n",
       "2  0000902 // cell morphogenesis // inferred from...   \n",
       "3  0001655 // urogenital system development // in...   \n",
       "4  0007165 // signal transduction // non-traceabl...   \n",
       "\n",
       "                    Gene Ontology Cellular Component  \\\n",
       "0  0005576 // extracellular region // inferred fr...   \n",
       "1  0005634 // nucleus // inferred from electronic...   \n",
       "2  0005737 // cytoplasm // inferred from direct a...   \n",
       "3  0005634 // nucleus // inferred from direct ass...   \n",
       "4  0001750 // photoreceptor outer segment // infe...   \n",
       "\n",
       "                    Gene Ontology Molecular Function  \n",
       "0  0000166 // nucleotide binding // inferred from...  \n",
       "1  0000166 // nucleotide binding // inferred from...  \n",
       "2  0000166 // nucleotide binding // inferred from...  \n",
       "3  0000979 // RNA polymerase II core promoter seq...  \n",
       "4  0005509 // calcium ion binding // inferred fro...  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpl = list(gse.gpls.values())[0].table;\n",
    "#gpl.head()\n",
    "gpl[[\"ID\", \"Gene Symbol\", \"Gene Ontology Biological Process\", \"Gene Ontology Cellular Component\", \"Gene Ontology Molecular Function\"]].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Show a Clustergram of Expression Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\seaborn\\matrix.py:654: UserWarning: Clustering large matrix with scipy. Installing `fastcluster` may give better performance.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 280024 characters]",
      "text/plain": [
       "<Figure size 720x720 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vars = gsedata.var(axis=1) #variances\n",
    "\n",
    "import numpy\n",
    "import math\n",
    "sortedvars = numpy.sort( vars.values )\n",
    "minvar = sortedvars[ math.floor(sortedvars.size*0.9) ]\n",
    "d2 = gsedata.drop( gsedata[vars<minvar].index, axis=0)\n",
    "\n",
    "import seaborn\n",
    "seaborn.set(style='darkgrid')\n",
    "clust = seaborn.clustermap(d2, metric=\"euclidean\", method=\"ward\", cmap=\"Blues\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b3ba2566441a7c06988d0923437866b63cedc61552a5af99d1f4fb67d367b25f"
  },
  "kernelspec": {
   "display_name": "walker3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
