{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"TidyAddr_EnochPratt.ipynb","provenance":[],"collapsed_sections":[],"authorship_tag":"ABX9TyNPKhM+ZUjDPmgVK9I1UAVS"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"puLNKyWEO6hi"},"source":["import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"K8Ph1dQhO5rR"},"source":["!git clone https://github.com/BNIA/tidyaddr-js.git"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cYs4OUGRPIrp"},"source":["! npm install tidyaddr-js/"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"f0TcSv9FPMRl"},"source":["cd tidyaddr-js"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"zmsaReRyOM6N"},"source":["pd.read_excel(open('EnochPratt_FY2020_VS19.xlsx', 'rb'), sheet_name='AllCardTypes2020').to_csv('lib.csv')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"VWf4O8wbPUyh"},"source":["ls"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"kX2yn1PQRymB"},"source":["!node tidyaddr.js clean-csv lib.csv tidyaddrd.csv"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"muVQ84pncrft"},"source":["df = pd.read_csv(\"tidyaddrd.csv\");"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1pLLsWcU2xFY"},"source":["df.head(1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"UHkmJP-twV9K"},"source":["df.to_excel('output1.xlsx', engine='xlsxwriter') ;\n","df.head(1)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Mor_eUfQc6Cb"},"source":["%%capture\n","! pip install geopy\n","! pip install geopandas\n","! pip install geoplot\n","! pip install dataplay"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uT4nqqZJMKIB"},"source":["ls"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"n7OAx0pnc5NU"},"source":["%%capture \n","# @title Run: Import Modules\n","\n","# These imports will handle everything\n","import os\n","import sys\n","import csv\n","from IPython.display import clear_output\n","import matplotlib.pyplot as plt\n","import numpy as np\n","import pandas as pd\n","import geopandas as gpd\n","from geopandas import GeoDataFrame\n","import psycopg2\n","import pyproj\n","from pyproj import Proj, transform\n","# conda install -c conda-forge proj4\n","from shapely.geometry import Point\n","from shapely.geometry import LineString\n","from shapely import wkb\n","from shapely.wkt import loads\n","# https://pypi.org/project/geopy/\n","from geopy.geocoders import Nominatim\n","import folium\n","from folium import plugins\n","\n","from dataplay.merge import mergeDatasets \n","\n","import dexplot as dxp\n","\n","# In case file is KML, enable support\n","import fiona\n","fiona.drvsupport.supported_drivers['kml'] = 'rw'\n","fiona.drvsupport.supported_drivers['KML'] = 'rw'"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"0XfmSweEctKL"},"source":["\n","address_df = df.copy()\n","addrCol = 'address'\n","\n","# An example of loading in an internal BNIA file\n","crs = {'init' :'epsg:2248'} "],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"K92GtQQZPSy3"},"source":["# More information vist: https://geopy.readthedocs.io/en/stable/#module-geopy.geocoders\n","\n","# In this example we retrieve and map a dataset with no lat/lng but containing an address\n","\n","# In this example our data is stored in the 'STREET' attribute\n","geometry = []\n","geolocator = Nominatim(user_agent=\"my-application\")\n","\n","for index, row in address_df.iterrows():\n","  # We will try and return an address for each Street Name\n","  try: \n","      # retrieve the geocoded information of our street address\n","      geol = geolocator.geocode(row[addrCol], timeout=None)\n","\n","      # create a mappable coordinate point from the response object's lat/lang values.\n","      pnt = Point(geol.longitude, geol.latitude)\n","      \n","      # Append this value to the list of geometries\n","      geometry.append(pnt)\n","      \n","  except: \n","      # If no street name was found decide what to do here.\n","      # df.loc[index]['geom'] = Point(0,0) # Alternate method\n","      geometry.append(Point(0,0))\n","      \n","# Finally, we stuff the geometry data we created back into the dataframe\n","address_df['geometry'] = geometry"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"AW9POzmhdhci"},"source":["# save after every 100th geocode"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bdfhYocwdjX7"},"source":[""],"execution_count":null,"outputs":[]}]}