{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7675ba93-ed1f-42e4-80ff-fb679f633ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4c29c09f-e01f-48a0-8751-89c536c345db",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f10e7cd8-2997-4417-8351-95bc9ff45652",
   "metadata": {},
   "source": [
    "# SDSS data available from: \n",
    "https://cdsarc.cds.unistra.fr/viz-bin/cat/J/MNRAS/455/2440#/browse"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "295c39ae-a1a7-4b75-b4db-fb838d082bee",
   "metadata": {},
   "source": [
    "---------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "673c361a-0902-4705-b323-8dd3f9922a15",
   "metadata": {},
   "source": [
    "Byte-by-byte&nbsp;Description&nbsp;of&nbsp;file:&nbsp;catalog.dat<br>--------------------------------------------------------------------------------<br>&nbsp;&nbsp;&nbsp;Bytes&nbsp;Format&nbsp;Units&nbsp;&nbsp;&nbsp;Label&nbsp;&nbsp;&nbsp;&nbsp;Explanations<br>--------------------------------------------------------------------------------<br>&nbsp;&nbsp;&nbsp;1-&nbsp;&nbsp;6&nbsp;&nbsp;I6&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Seq&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Galaxy&nbsp;number&nbsp;(galcount)<br>&nbsp;&nbsp;&nbsp;8-&nbsp;25&nbsp;&nbsp;I18&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;objID&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;photometric&nbsp;object&nbsp;ID&nbsp;(objid)<br>&nbsp;&nbsp;27-&nbsp;30&nbsp;&nbsp;A4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[SDSS]<br>&nbsp;&nbsp;32-&nbsp;50&nbsp;&nbsp;A19&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;IAU&nbsp;designation&nbsp;of&nbsp;the&nbsp;source<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(JHHMMSS.ss+DDMMSS.s)&nbsp;(SDSSIAU)<br>&nbsp;&nbsp;52-&nbsp;53&nbsp;&nbsp;I2&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;BadFl&nbsp;&nbsp;&nbsp;&nbsp;[0/84]&nbsp;SDSS&nbsp;photometric&nbsp;pipeline&nbsp;flags<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SATURATED,&nbsp;BRIGHT,&nbsp;EDGE,&nbsp;NODEBLEND,&nbsp;CHILD,&nbsp;and<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;DEBLENDED&nbsp;AS&nbsp;PSF&nbsp;(badflag)<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;55&nbsp;&nbsp;I1&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Nch&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0]&nbsp;Number&nbsp;of&nbsp;child&nbsp;sources&nbsp;deblended&nbsp;by<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;the&nbsp;SDSS&nbsp;(nchild)<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;57&nbsp;&nbsp;I1&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Mode&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[1/2]&nbsp;Mode&nbsp;(1:&nbsp;primary;&nbsp;2:&nbsp;secondary)&nbsp;(mode)<br>&nbsp;&nbsp;59-&nbsp;62&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;run&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;run&nbsp;number&nbsp;(run)<br>&nbsp;&nbsp;64-&nbsp;66&nbsp;&nbsp;I3&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;rerun&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;rerun&nbsp;number&nbsp;(rerun)<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;68&nbsp;&nbsp;I1&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;camCol&nbsp;&nbsp;&nbsp;SDSS&nbsp;camCol&nbsp;number&nbsp;(camCol)<br>&nbsp;&nbsp;70-&nbsp;72&nbsp;&nbsp;I3&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;field&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;field&nbsp;number&nbsp;(field)<br>&nbsp;&nbsp;74-&nbsp;77&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;obj&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;object&nbsp;number&nbsp;(obj)<br>&nbsp;&nbsp;79-&nbsp;82&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;stripe&nbsp;&nbsp;&nbsp;SDSS&nbsp;stripe&nbsp;number&nbsp;(stripe)<br>&nbsp;&nbsp;84-&nbsp;90&nbsp;&nbsp;I7&nbsp;&nbsp;&nbsp;&nbsp;arcsec&nbsp;&nbsp;startmu&nbsp;&nbsp;SDSS&nbsp;starting&nbsp;mu&nbsp;for&nbsp;the&nbsp;observation&nbsp;of<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;the&nbsp;chunk&nbsp;(startmu)<br>&nbsp;&nbsp;92-109&nbsp;&nbsp;I18&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;specobjid&nbsp;Unique&nbsp;SDSS&nbsp;spectroscopic&nbsp;id&nbsp;number&nbsp;(specobjid)<br>&nbsp;111-114&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Plate&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;plate&nbsp;number&nbsp;(plate)<br>&nbsp;116-120&nbsp;&nbsp;I5&nbsp;&nbsp;&nbsp;&nbsp;MJD&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;MJD&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;Modified&nbsp;Julian&nbsp;Date&nbsp;of&nbsp;observation&nbsp;(mjd)<br>&nbsp;122-124&nbsp;&nbsp;I3&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Fiber&nbsp;&nbsp;&nbsp;&nbsp;SDSS&nbsp;Fiber&nbsp;number&nbsp;(fiberid)<br>&nbsp;126-135&nbsp;&nbsp;F10.6&nbsp;deg&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;RAdeg&nbsp;&nbsp;&nbsp;&nbsp;Right&nbsp;ascension&nbsp;(J2000)&nbsp;(ra)<br>&nbsp;137-146&nbsp;&nbsp;F10.6&nbsp;deg&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;DEdeg&nbsp;&nbsp;&nbsp;&nbsp;Declination&nbsp;(J2000)&nbsp;(dec)<br>&nbsp;148-155&nbsp;&nbsp;F8.6&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;z&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0.005/0.9]&nbsp;Redshift&nbsp;of&nbsp;the&nbsp;galaxy&nbsp;(z)<br>&nbsp;157-168&nbsp;&nbsp;F12.6&nbsp;km/s&nbsp;&nbsp;&nbsp;&nbsp;sigma&nbsp;&nbsp;&nbsp;&nbsp;[11/415]?=-&nbsp;SDSS&nbsp;measured&nbsp;velocity&nbsp;dispersion<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(veldisp)<br>&nbsp;170-181&nbsp;&nbsp;F12.6&nbsp;km/s&nbsp;&nbsp;e_sigma&nbsp;&nbsp;&nbsp;&nbsp;[1/177]?=-&nbsp;rms&nbsp;uncertainty&nbsp;on&nbsp;sigma<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(veldispErr)<br>&nbsp;183-191&nbsp;&nbsp;F9.6&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;eclass&nbsp;&nbsp;&nbsp;[-1/1.2]&nbsp;SDSS&nbsp;spectroscopic&nbsp;classification<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(eclass)<br>&nbsp;193-200&nbsp;&nbsp;F8.3&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;peld&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-888/1]&nbsp;Galaxy&nbsp;Zoo&nbsp;debiased&nbsp;probability&nbsp;of<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;elliptical&nbsp;galaxy&nbsp;(p_el_debiased)<br>&nbsp;202-209&nbsp;&nbsp;F8.3&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;pcsd&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-888/1]&nbsp;Galaxy&nbsp;Zoo&nbsp;debiased&nbsp;probability&nbsp;of<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;spiral&nbsp;galaxy&nbsp;(p_cs_debiased)<br>&nbsp;211-214&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;NS&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-888/1]&nbsp;The&nbsp;raw&nbsp;Galaxy&nbsp;Zoo&nbsp;votes&nbsp;for&nbsp;spiral<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;type&nbsp;(spiral)<br>&nbsp;216-219&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;NE&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-888/1]&nbsp;The&nbsp;raw&nbsp;Galaxy&nbsp;Zoo&nbsp;votes&nbsp;for<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;elliptical&nbsp;type&nbsp;(elliptical)<br>&nbsp;221-224&nbsp;&nbsp;I4&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;NU&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-888/1]&nbsp;The&nbsp;raw&nbsp;Galaxy&nbsp;Zoo&nbsp;votes&nbsp;for&nbsp;uncertain<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;type&nbsp;(uncertain)<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bef8fd80-0981-4d75-9a7b-14f6e9e71167",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['Seq','objID','SDSS','BadFl','Nch','Mode','run','rerun','camCol','field','obj','stripe','startmu','specobjid','Plate','MJD','Fiber','RAdeg','DEdeg','z','sigma','e_sigma','eclass','peld','pcsd','NS','NE','NU']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ae7e2a0-eb0e-4622-a2aa-e3f1ed76a36b",
   "metadata": {},
   "outputs": [],
   "source": [
    "SDSS_df = pd.read_csv(\"/Users/agurvich/src/catalog.dat\",delimiter='|',header=None,names=labels,usecols=['RAdeg','DEdeg','z','peld','pcsd','NS','NE','NU'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dae8919-fb03-4ba1-b1e4-5942e292ef0e",
   "metadata": {},
   "source": [
    "Byte-by-byte&nbsp;Description&nbsp;of&nbsp;file:&nbsp;nonparam*.dat<br>--------------------------------------------------------------------------------<br>&nbsp;&nbsp;&nbsp;Bytes&nbsp;Format&nbsp;Units&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Label&nbsp;&nbsp;Explanations<br>--------------------------------------------------------------------------------<br>1-&nbsp;&nbsp;8&nbsp;&nbsp;F8.3&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;[12.4/98]?=-&nbsp;SExtractor&nbsp;magnitude&nbsp;(SexMag)<br>&nbsp;&nbsp;10-&nbsp;18&nbsp;&nbsp;F9.3&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;e_mag&nbsp;&nbsp;&nbsp;&nbsp;[-9999.99/99]?=-&nbsp;mag&nbsp;error&nbsp;(SexMag_Err)<br>&nbsp;&nbsp;20-&nbsp;28&nbsp;&nbsp;F9.4&nbsp;&nbsp;arcsec&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;rh&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-3/51]?=-&nbsp;SExtractor&nbsp;half-light&nbsp;radii<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(SexHrad)<br>&nbsp;&nbsp;30-&nbsp;38&nbsp;&nbsp;F9.4&nbsp;mag/arcsec2&nbsp;&nbsp;&nbsp;Sky&nbsp;&nbsp;&nbsp;&nbsp;[19/23]?=-&nbsp;SExtractor&nbsp;sky&nbsp;brightness<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(SexSky)<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;40&nbsp;&nbsp;I1&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Nt&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0/4]&nbsp;The&nbsp;number&nbsp;of&nbsp;targets&nbsp;(num_targets)<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;42&nbsp;&nbsp;I1&nbsp;&nbsp;&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Nn&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0/8]&nbsp;The&nbsp;number&nbsp;neighbour&nbsp;sources&nbsp;fitted<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;with&nbsp;Ser&nbsp;profiles&nbsp;(always&nbsp;\"0\"&nbsp;for&nbsp;g-&nbsp;and<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;i-&nbsp;bands)&nbsp;(num_neighborfit)<br>&nbsp;&nbsp;44-&nbsp;50&nbsp;&nbsp;F7.2&nbsp;&nbsp;arcsec&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Rad&nbsp;&nbsp;&nbsp;&nbsp;[1.8/238]?=-&nbsp;The&nbsp;aperture&nbsp;radius&nbsp;used&nbsp;for<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;colour&nbsp;measurements<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(always&nbsp;null&nbsp;for&nbsp;g-band)&nbsp;(AperRad)<br>&nbsp;&nbsp;52-&nbsp;58&nbsp;&nbsp;F7.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;C&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-2.8/7.2]?=-&nbsp;The&nbsp;concentration&nbsp;(C)<br>&nbsp;&nbsp;60-&nbsp;68&nbsp;&nbsp;E9.4&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;e_C&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0.0001/4.2e+11]?=-&nbsp;Concentration&nbsp;error<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(C_err)<br>&nbsp;&nbsp;70-&nbsp;77&nbsp;&nbsp;F8.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-5826/1.9]?=-&nbsp;The&nbsp;asymmetry&nbsp;(A)<br>&nbsp;&nbsp;79-&nbsp;87&nbsp;&nbsp;E9.4&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;e_A&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0.001/5.7e+07]?=-&nbsp;Asymmetry&nbsp;error&nbsp;(A_err)<br>&nbsp;&nbsp;89-&nbsp;96&nbsp;&nbsp;F8.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;S&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-7215/14175]?=-&nbsp;The&nbsp;smoothness&nbsp;(S)<br>&nbsp;&nbsp;98-106&nbsp;&nbsp;E9.4&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;e_S&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0.001/1.2e+09]?=-&nbsp;Smoothness&nbsp;error&nbsp;(S_err)<br>&nbsp;108-114&nbsp;&nbsp;F7.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;G&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[0.2/0.9]?=-&nbsp;The&nbsp;Gini&nbsp;coefficient&nbsp;(G)<br>&nbsp;116-122&nbsp;&nbsp;F7.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;M20&nbsp;&nbsp;&nbsp;&nbsp;[-3.7/1.2]?=-&nbsp;The&nbsp;M20&nbsp;value&nbsp;(M20)<br>&nbsp;124-128&nbsp;&nbsp;F5.3&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Ext&nbsp;&nbsp;&nbsp;&nbsp;[0.006/1.5]&nbsp;The&nbsp;SDSS-provided&nbsp;galactic<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;extinction&nbsp;(extinction)<br>&nbsp;130-135&nbsp;&nbsp;F6.3&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;DM&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[31/44]&nbsp;The&nbsp;calculated&nbsp;distance&nbsp;modulus<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(dismod)<br>&nbsp;137-140&nbsp;&nbsp;F4.2&nbsp;kpc/arcsec&nbsp;&nbsp;&nbsp;&nbsp;Scale&nbsp;&nbsp;[0.1/8]&nbsp;The&nbsp;angular&nbsp;scale&nbsp;(kpc_per_arcsec)<br>&nbsp;142-151&nbsp;&nbsp;E10.4&nbsp;Mpc3&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Vmax&nbsp;&nbsp;&nbsp;[-1.8e+09/3e+10]&nbsp;The&nbsp;volume&nbsp;used&nbsp;for&nbsp;Vmax<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;corrections&nbsp;(Vmax)<br>&nbsp;153-159&nbsp;&nbsp;F7.2&nbsp;&nbsp;---&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;SN&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[-1.2/1270]?=-&nbsp;The&nbsp;average&nbsp;S/N&nbsp;per&nbsp;pixel<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;inside&nbsp;the&nbsp;half-light&nbsp;radius<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(only&nbsp;for&nbsp;r-band)&nbsp;(SN)<br>&nbsp;161-167&nbsp;&nbsp;F7.2&nbsp;&nbsp;mag&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;kcorr&nbsp;&nbsp;[-1.6/2.3]?=-&nbsp;k-correction&nbsp;calculated&nbsp;using<br>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;the&nbsp;SDSS&nbsp;model&nbsp;magnitudes&nbsp;(kcorr)<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bccadd5b-9ba4-435c-b426-15233a7613c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "nonparam_labels = ['mag','e_mag','rh','2Sky','Nt','Nn','Rad','C','e_C','A','e_A','S','e_S','G','M20','Ext','DM','Scale','Vmax','SN','kcorr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0fa405c-33f0-48bf-8d6e-1779e2860cfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_abs_mag(band_df):\n",
    "    app_mag = band_df['mag'].astype(float)\n",
    "    print(app_mag.size - app_mag.dropna().size)\n",
    "    app_mag[app_mag.isna()] = np.nanmean(app_mag)\n",
    "    DM = band_df['DM'].astype(float)\n",
    "    Ext = band_df['Ext'].astype(float)\n",
    "    kcorr = band_df['kcorr'].astype(float)\n",
    "    print(kcorr.size - kcorr.dropna().size)\n",
    "    kcorr[kcorr.isna()] = np.nanmean(kcorr)\n",
    "    return app_mag-DM-Ext-kcorr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "596774fe-93e8-480d-a025-cb8724ccec24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/87/2kv01skd4471fn2y8rcqj6700000gn/T/ipykernel_15331/929029020.py:2: DtypeWarning: Columns (20) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  band_df = pd.read_csv(f'/Users/agurvich/src/nonparam{band}.dat',delimiter='|',names=nonparam_labels)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "463\n",
      "473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/87/2kv01skd4471fn2y8rcqj6700000gn/T/ipykernel_15331/929029020.py:2: DtypeWarning: Columns (20) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  band_df = pd.read_csv(f'/Users/agurvich/src/nonparam{band}.dat',delimiter='|',names=nonparam_labels)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1645\n",
      "4447\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/87/2kv01skd4471fn2y8rcqj6700000gn/T/ipykernel_15331/929029020.py:2: DtypeWarning: Columns (20) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  band_df = pd.read_csv(f'/Users/agurvich/src/nonparam{band}.dat',delimiter='|',names=nonparam_labels)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1337\n",
      "165426\n"
     ]
    }
   ],
   "source": [
    "for band in ['r','g','i']:\n",
    "    band_df = pd.read_csv(f'/Users/agurvich/src/nonparam{band}.dat',delimiter='|',names=nonparam_labels)\n",
    "    for key in band_df.keys():\n",
    "        if band_df[key].dtype != 'object': continue\n",
    "        band_df.loc[band_df[key].astype(str).str.contains('-'),key] = np.nan\n",
    "    SDSS_df[f'mag_{band}'] = get_abs_mag(band_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "51f5a0c2-c895-4ecd-baed-a3961d126464",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RAdeg</th>\n",
       "      <th>DEdeg</th>\n",
       "      <th>z</th>\n",
       "      <th>peld</th>\n",
       "      <th>pcsd</th>\n",
       "      <th>NS</th>\n",
       "      <th>NE</th>\n",
       "      <th>NU</th>\n",
       "      <th>mag_r</th>\n",
       "      <th>mag_g</th>\n",
       "      <th>mag_i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>236.246994</td>\n",
       "      <td>-0.475264</td>\n",
       "      <td>0.030284</td>\n",
       "      <td>0.165</td>\n",
       "      <td>0.739</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-18.246</td>\n",
       "      <td>-17.747</td>\n",
       "      <td>-18.426328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>236.287003</td>\n",
       "      <td>-0.518003</td>\n",
       "      <td>0.077923</td>\n",
       "      <td>0.029</td>\n",
       "      <td>0.937</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-20.656</td>\n",
       "      <td>-20.255</td>\n",
       "      <td>-20.922328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>236.341995</td>\n",
       "      <td>-0.467023</td>\n",
       "      <td>0.160228</td>\n",
       "      <td>0.211</td>\n",
       "      <td>0.763</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-22.771</td>\n",
       "      <td>-22.403</td>\n",
       "      <td>-22.820000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>236.350006</td>\n",
       "      <td>-0.598237</td>\n",
       "      <td>0.071558</td>\n",
       "      <td>0.171</td>\n",
       "      <td>0.787</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-20.985</td>\n",
       "      <td>-20.381</td>\n",
       "      <td>-21.279000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>236.369003</td>\n",
       "      <td>-0.574450</td>\n",
       "      <td>0.071975</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.981</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-21.269</td>\n",
       "      <td>-20.716</td>\n",
       "      <td>-21.598328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670717</th>\n",
       "      <td>217.363007</td>\n",
       "      <td>1.049700</td>\n",
       "      <td>0.086556</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-21.691</td>\n",
       "      <td>-21.023</td>\n",
       "      <td>-22.076000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670718</th>\n",
       "      <td>226.462997</td>\n",
       "      <td>1.074650</td>\n",
       "      <td>0.087805</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-21.511</td>\n",
       "      <td>-21.202</td>\n",
       "      <td>-21.864328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670719</th>\n",
       "      <td>226.546997</td>\n",
       "      <td>1.114430</td>\n",
       "      <td>0.104590</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-23.011</td>\n",
       "      <td>-22.150</td>\n",
       "      <td>-23.387000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670720</th>\n",
       "      <td>226.595993</td>\n",
       "      <td>1.148130</td>\n",
       "      <td>0.126815</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-21.352</td>\n",
       "      <td>-20.936</td>\n",
       "      <td>-21.668328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670721</th>\n",
       "      <td>259.963013</td>\n",
       "      <td>26.509300</td>\n",
       "      <td>0.105541</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888.000</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-888</td>\n",
       "      <td>-20.912</td>\n",
       "      <td>-20.282</td>\n",
       "      <td>-21.889242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>670722 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             RAdeg      DEdeg         z     peld     pcsd   NS   NE   NU  \\\n",
       "0       236.246994  -0.475264  0.030284    0.165    0.739    0    0    1   \n",
       "1       236.287003  -0.518003  0.077923    0.029    0.937    1    0    0   \n",
       "2       236.341995  -0.467023  0.160228    0.211    0.763    0    0    1   \n",
       "3       236.350006  -0.598237  0.071558    0.171    0.787    1    0    0   \n",
       "4       236.369003  -0.574450  0.071975    0.019    0.981    1    0    0   \n",
       "...            ...        ...       ...      ...      ...  ...  ...  ...   \n",
       "670717  217.363007   1.049700  0.086556    1.000    0.000    0    1    0   \n",
       "670718  226.462997   1.074650  0.087805 -888.000 -888.000 -888 -888 -888   \n",
       "670719  226.546997   1.114430  0.104590 -888.000 -888.000 -888 -888 -888   \n",
       "670720  226.595993   1.148130  0.126815 -888.000 -888.000 -888 -888 -888   \n",
       "670721  259.963013  26.509300  0.105541 -888.000 -888.000 -888 -888 -888   \n",
       "\n",
       "         mag_r   mag_g      mag_i  \n",
       "0      -18.246 -17.747 -18.426328  \n",
       "1      -20.656 -20.255 -20.922328  \n",
       "2      -22.771 -22.403 -22.820000  \n",
       "3      -20.985 -20.381 -21.279000  \n",
       "4      -21.269 -20.716 -21.598328  \n",
       "...        ...     ...        ...  \n",
       "670717 -21.691 -21.023 -22.076000  \n",
       "670718 -21.511 -21.202 -21.864328  \n",
       "670719 -23.011 -22.150 -23.387000  \n",
       "670720 -21.352 -20.936 -21.668328  \n",
       "670721 -20.912 -20.282 -21.889242  \n",
       "\n",
       "[670722 rows x 11 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SDSS_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6900a4d-ebc9-4d30-bbe6-20aab101a87e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from astropy.coordinates import SkyCoord\n",
    "from astropy.cosmology import Planck18 as cosmo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "289c5484-7c80-4b99-aa10-b2d71e4c9b5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "radec_coords = SkyCoord(SDSS_df.RAdeg,SDSS_df.DEdeg,unit='deg')\n",
    "coords = np.zeros((radec_coords.size,3))\n",
    "coords[:,0] = radec_coords.cartesian.x\n",
    "coords[:,1] = radec_coords.cartesian.y\n",
    "coords[:,2] = radec_coords.cartesian.z\n",
    "\n",
    "\n",
    "dists = np.array(cosmo.luminosity_distance(SDSS_df.z))\n",
    "dists/=np.nanmean(dists)\n",
    "\n",
    "coords*= dists[:,None]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b90b39c-4958-4f0b-88b0-fe1aeac52fd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ad152a22-7b28-4485-880d-5bd19fd454ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5fe17f17-ae41-44b7-948b-176b6b76306d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x17bd72520>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 15912 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(coords[:,0],coords[:,1],'.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0c52562c-8c95-4df8-8a4e-8d1ee54eba61",
   "metadata": {},
   "outputs": [],
   "source": [
    "field_names = ['z','mag_r','mag_g','mag_i','pcsd','peld']\n",
    "fields = np.zeros((len(field_names),coords.shape[0]))\n",
    "\n",
    "for i,key in enumerate(field_names):\n",
    "    fields[i,:] = SDSS_df[key]\n",
    "field_names[0] = 'redshift'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e36e3dc3-39f0-4f69-ba23-247298585c3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb_r = 10**(-0.4*np.array(SDSS_df.mag_i-4.57)) ## solar luminosities\n",
    "rgb_g = 10**(-0.4*np.array(SDSS_df.mag_r-4.67)) ## solar luminosities\n",
    "rgb_b = 10**(-0.4*np.array(SDSS_df.mag_g-5.12)) ## solar luminosities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3e14cfff-e535-4df1-9357-974596eadbe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "good_mask = np.logical_and(\n",
    "    ~np.any(np.isnan(coords),axis=1),\n",
    "    np.isfinite(rgb_r+rgb_g+rgb_b))\n",
    "coords = coords[good_mask]\n",
    "fields = fields[...,good_mask]\n",
    "rgb_r = rgb_r[good_mask]\n",
    "rgb_g = rgb_g[good_mask]\n",
    "rgb_b = rgb_b[good_mask]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6faced0d-30ba-4a06-b6f6-d50e50b416a8",
   "metadata": {},
   "source": [
    "### need to figure out what the dynamic range of the luminosities is in order to remap to RGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b15eb779-104f-4ceb-99aa-e1abb1d32f7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.lines.Line2D at 0x17c0a22e0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 13816 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lums = np.empty(1)\n",
    "for band,color in zip([rgb_r,rgb_g,rgb_b],'rgb'):\n",
    "    plt.hist(np.log10(band),bins=np.linspace(7.8,11.5),color=color,histtype='step',lw=3)\n",
    "    lums = np.append(lums,band)\n",
    "    \n",
    "plt.hist(np.log10(lums),bins=np.linspace(7.8,11.5),color='k',histtype='step',lw=3,ls=':')\n",
    "bottom = np.percentile(lums,1)\n",
    "top = np.percentile(lums,99)\n",
    "ax = plt.gca()\n",
    "ax.axvline(np.log10(bottom),ls='--',c='k',lw=1.5)\n",
    "ax.axvline(np.log10(top),ls='--',c='k',lw=1.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2a4c962-626f-4ed5-9370-6e793d14aeb7",
   "metadata": {},
   "source": [
    "### goes from ~1e9 to ~1e11 solar luminosities, seems reasonable for a bunch of galaxies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "65dbf021-5877-4cb6-9226-de90076cb776",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 8312 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rgbas = np.zeros((rgb_r.shape[0],4))\n",
    "for i,channel in enumerate([rgb_r,rgb_g,rgb_b]):\n",
    "#    bottom = top = np.percentile(channel,5)\n",
    "#    top = np.percentile(channel,99)\n",
    "    rgbas[:,i] = np.log10(channel/bottom) / np.log10(top/bottom)\n",
    "    \n",
    "    ## clamp the values between 0 and 1\n",
    "    rgbas[:,i][rgbas[:,i]<0] = 0\n",
    "    rgbas[:,i][rgbas[:,i]>1] = 1\n",
    "\n",
    "## set the alpha values to 1\n",
    "rgbas[:,-1] = 1\n",
    "\n",
    "## look at the resulting colors\n",
    "plt.hist(rgbas[:,0],histtype='step',lw=3,color='r',bins=np.linspace(0,1))\n",
    "plt.hist(rgbas[:,1],histtype='step',lw=3,color='g',bins=np.linspace(0,1))\n",
    "plt.hist(rgbas[:,2],histtype='step',lw=3,color='b',bins=np.linspace(0,1))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "39f82829-0d0b-4bf1-b337-739f747d9df1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12064 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(\n",
    "    fields[-2,:][fields[-1,:]<0.5],\n",
    "    bins=np.linspace(0,1),\n",
    "    color='b',\n",
    "    histtype='step',\n",
    "    lw=3,\n",
    "    label=r\"$P(D|P(E)<0.5)$\")\n",
    "plt.hist(\n",
    "    fields[-1,:][fields[-2,:]<0.5],\n",
    "    bins=np.linspace(0,1),\n",
    "    color='r',\n",
    "    histtype='step',\n",
    "    lw=3,\n",
    "    label=r\"$P(E|P(D)<0.5)$\")\n",
    "ax = plt.gca()\n",
    "ax.axvline(0.5,ls='--',c='k',lw=1.5)\n",
    "ax.text(0.2,1.5e4,'uncertain')\n",
    "ax.text(0.6,1.5e4,'likely D or E')\n",
    "ax.legend(loc=2,frameon=False)\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de737bad-d7dd-432c-b4c3-915a26cc693c",
   "metadata": {},
   "source": [
    "# Format Data for Firefly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2ae6e67d-b2dc-4fda-93c3-c2df2c9e39fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "palettable colormaps are not installed\n",
      "don't have phil's colormaps\n"
     ]
    }
   ],
   "source": [
    "from firefly.data_reader import ArrayReader,ParticleGroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1a93004b-1cdd-4bf9-a7a5-e561543ee7c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "JSONdir: /Users/agurvich/SDSS -- is not a sub-directory of firefly/static/data. \n",
      "This may produce confusing or inoperable results. As such, we will create a symlink for you when you  dumpToJSON.\n",
      "Make sure each field_array (6) has a field_filter_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_colormap_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_radius_flag (0), assuming False.\n",
      "Outputting: Full - 670722/670722 particles - 6 tracked fields\n"
     ]
    }
   ],
   "source": [
    "my_arrayReader = ArrayReader(\n",
    "    coords,\n",
    "    JSONdir='SDSS',\n",
    "    UInames=['Full'],\n",
    "    fields=fields,\n",
    "    field_names=field_names,\n",
    "    write_to_disk=False,\n",
    "    write_startup=True,\n",
    "    clean_JSONdir=True)\n",
    "my_arrayReader.particleGroups[0].rgba_colors = rgbas\n",
    "my_arrayReader.settings['camera'] = [0,0,-300]\n",
    "my_arrayReader.settings['sizeMult']['Full'] = .1\n",
    "my_arrayReader.settings['showParts']['Full'] = False\n",
    "my_arrayReader.settings['color']['Full'] = [1,1,1,1]\n",
    "my_arrayReader.settings['UIcolorPicker']['Full'] = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bf943c1-70d9-4363-9f38-186efde2597c",
   "metadata": {},
   "source": [
    "## Let's make particle groups for the different morphological classifications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "421427b8-2662-47b9-8ad4-5a115ee0075c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.colors import to_rgba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3beb5022-f970-42a9-a6ae-cbcd9867ed94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make sure each field_array (6) has a field_filter_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_colormap_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_radius_flag (0), assuming False.\n",
      "Make sure each field_array (6) has a field_filter_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_colormap_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_radius_flag (0), assuming False.\n",
      "Make sure each field_array (6) has a field_filter_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_colormap_flag (0), assuming True.\n",
      "Make sure each field_array (6) has a field_radius_flag (0), assuming False.\n"
     ]
    }
   ],
   "source": [
    "## likely disks\n",
    "UInames = ['Disks','Ellipticals','Uncertain'][::-1]\n",
    "colors = ['#4477AA', '#CC6677', '#DDCC77'][::-1]\n",
    "pds,pes = fields[-2:]\n",
    "masks = [\n",
    "    (pds >= 0.5) & (pes <= 0.5),\n",
    "    (pes >= 0.5) & (pds < 0.5),\n",
    "    (pes < 0.5) & (pds < 0.5),\n",
    "][::-1]\n",
    "if (masks[0].size - np.sum(masks)) != 0:\n",
    "    raise ValueError(\"Not a correct partitioning\")\n",
    "    \n",
    "for i,mask in enumerate(masks):\n",
    "    pg = ParticleGroup(\n",
    "        UInames[i],\n",
    "        coords[mask],\n",
    "        None,\n",
    "        field_arrays=fields[...,mask],\n",
    "        field_names=field_names,color=to_rgba(colors[i]),\n",
    "        sizeMult=.15)\n",
    "    my_arrayReader.addParticleGroup(pg) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4870349a-aa26-4762-960b-be775cc0abf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ArrayReader with 4 particle groups\n",
      "[Full - 670722/670722 particles - 6 tracked fields\n",
      " Uncertain - 106795/106795 particles - 6 tracked fields\n",
      " Ellipticals - 252715/252715 particles - 6 tracked fields\n",
      " Disks - 311212/311212 particles - 6 tracked fields]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(my_arrayReader)\n",
    "print(my_arrayReader.particleGroups)\n",
    "my_arrayReader.writeToDisk()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bf1af89-0f38-4550-a8e6-117e31dd7c74",
   "metadata": {},
   "source": [
    "# automatically create repository and initialize github pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "da07b968-4df0-474e-abdc-6d7dc906ccaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/Users/agurvich/SDSS_test']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_arrayReader.copyFireflySourceToTarget('SDSS_test',init_gh_pages=False,overwrite=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
