# AUTOGENERATED! DO NOT EDIT! File to edit: notebooks/Mdprop_Permits_Fore_Vac Vio.ipynb (unless otherwise specified).

__all__ = ['totalres', 'totalres', 'totalres', 'totalres', 'totalres', 'totalres', 'ownroc', 'ownroc', 'ownroc',
           'ownroc', 'ownroc', 'ownroc', 'ownroc', 'ownroc', 'vacantsCsa', 'vacantsCsa', 'vacantsCsa', 'vacantsCsa',
           'vacantsCsa', 'baltvac', 'baltvac', 'baltvac', 'baltvac', 'baltvac', 'baltvac', 'vio', 'vio', 'start_date',
           'end_date', 'mask', 'mask1', 'mask2', 'vio', 'vio', 'vio', 'fore', 'fore', 'fore', 'resrehab', 'use',
           'resrehab', 'resrehab', 'resrehab', 'resrehab', 'demper', 'filter', 'demper', 'demper', 'demper', 'demper',
           'use', 'constper', 'constper', 'constper', 'constper', 'constper', 'constper', 'comprop', 'comprop',
           'comprop', 'comprop', 'comprop', 'crehab', 'crehab', 'crehab', 'crehab', 'crehab', 'crehab', 'crehab']

# Cell
totalres = mdprop.copy()

totalres['totalres'+year] = 1
totalres = totalres[ totalres['DESCLU'].isin(['Apartments', 'Residential', 'Residential Commercial', 'Residential Condominium']) ]
totalres = totalres[ totalres['ADDRESS'].notna() ]
print(totalres.ADDRESS.unique() )
totalres = totalres[['CSA2010','totalres'+year]]
totalres = totalres.groupby('CSA2010').sum(numeric_only=True)
# Make sure ALL csas and BaltimoreCity are included. among other things
totalres = csa[ ['CSA2010','tpop10'] ].merge( totalres, left_on='CSA2010', right_on='CSA2010', how='outer' )
# Update the baltimore CSA.
totalres.at[55,'totalres'+year] = totalres['totalres'+year].sum()

# Create the Indicator
totalres.to_csv('37-totalres-'+year+'.csv', index=False)

totalres.head(58)

# Cell
import datetime
ownroc = mdprop.copy()

ownroc = ownroc[ ownroc['OOI']=='H']
ownroc = ownroc.dropna( subset=['ADDRESS'] )
ownroc = ownroc[ ownroc['DESCLU'].isin(['Apartments', 'Residential', 'Residential Commercial', 'Residential Condominium']) ]
ownroc.to_csv('ownroc'+str(year)+'_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', ownroc.size / len(ownroc.columns) )

# Aggregate Numeric Values by Sum
ownroc['ownrocCount'] = 1
ownroc = ownroc.groupby('CSA2010').sum(numeric_only=True)
ownroc = csa[ ['CSA2010'] ].merge( ownroc, left_on='CSA2010', right_on='CSA2010', how='outer' )
ownroc = csa.merge( ownroc, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Create the Indicator
ownroc['ownroc'] = ownroc['ownrocCount'] * 100 / totalres['totalres'+year]
ownroc.at[55,'ownrocCount'] = ownroc['ownrocCount'].sum()
ownroc.at[55,'ownroc'] = ownroc['ownrocCount'].sum() * 100/ totalres['totalres'+year].sum()
ownroc = ownroc[ ['CSA2010', 'ownrocCount', 'ownroc'] ]
ownroc.to_csv('32-ownroc'+year+'.csv', index=False)

ownroc.tail(60)

# Cell
import datetime
vacantsCsa = vacants.copy()

# (datenotice between '2004-01-01' and '2016-12-31') AND
# (dateabate is NULL OR dateabate >= '2016-12-31') AND
# (datecancel is NULL OR datecancel > '2016-12-31')

vacantsCsa['DateNotice2'] = pd.to_datetime(vacantsCsa['DateNotice'],infer_datetime_format=True)
vacantsCsa = vacantsCsa[
    ( vacantsCsa['DateNotice2']>=pd.Timestamp(2000+int(year)-13,1,1) ) &
    ( vacantsCsa['DateNotice2']<=pd.Timestamp(2000+int(year),12,31) )
]

vacantsCsa.to_csv('vacants_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', vacantsCsa.size / len(vacantsCsa.columns) )

# Aggregate Numeric Values by Sum
vacantsCsa['vacantsCount'] = 1
vacantsCsa = vacantsCsa.groupby('CSA2010').sum(numeric_only=True)
vacantsCsa = totalres[ ['CSA2010', 'totalres'+year] ].merge( vacantsCsa, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Create the Indicator
vacantsCsa['vacants'+year] = vacantsCsa['vacantsCount'] * 100 / totalres['totalres'+year]
vacantsCsa.at[55,'vacantsCount'] = vacantsCsa['vacantsCount'].sum()
vacantsCsa.at[55,'vacants'+year] = vacantsCsa['vacantsCount'].sum() * 100 / totalres['totalres'+year].sum()
vacantsCsa = vacantsCsa[ ['CSA2010', 'vacantsCount', 'vacants'+year, 'totalres'+year ] ]
vacantsCsa.to_csv('34-vacants'+year+'.csv', index=False)

vacantsCsa.tail(60)

# Cell
import datetime
baltvac = vacants.copy()

baltvac = baltvac[ (baltvac['OwnerAbbr'].str.contains('DHCD|HABC|HUD|MCC|USA', regex=True, na=False) ) ]
baltvac['DateNotice2'] = pd.to_datetime(baltvac['DateNotice'],infer_datetime_format=True)
baltvac = baltvac[
    ( baltvac['DateNotice2']>=pd.Timestamp(2000+int(year)-13,1,1) ) &
    ( baltvac['DateNotice2']<=pd.Timestamp(2000+int(year),12,31) )
]

baltvac.to_csv('baltvac_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', baltvac.size / len(baltvac.columns) )

# Aggregate Numeric Values by Sum
baltvac['baltvacCount'] = 1
baltvac = baltvac.groupby('CSA2010').sum(numeric_only=True)
baltvac = vacantsCsa[ ['CSA2010', 'vacants'+year] ].merge( baltvac, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Create the Indicator
baltvac['baltvac'+year] = baltvac['baltvacCount'] / vacantsCsa['vacantsCount'] * 100
baltvac.at[55,'baltvacCount'] = baltvac['baltvacCount'].sum()
baltvac.at[55,'baltvac'+year] = baltvac['baltvacCount'].sum() * 100 / vacantsCsa['vacantsCount'].sum()
baltvac = baltvac[ ['CSA2010', 'baltvacCount', 'baltvac'+year, 'vacants'+year ] ]
baltvac.to_csv('43-baltvac'+year+'.csv', index=False)

baltvac.tail(60)

# Cell
# Numerator
vio = violations.copy()

# drop null
vio['DateCancel'] = pd.to_datetime(vio['DateCancel'])
vio['DateAbate'] = pd.to_datetime(vio['DateAbate'])
vio['DateNotice'] = pd.to_datetime(vio['DateNotice'], errors='coerce')

# Numerator
vio = vio[['DateNotice', 'DateAbate', 'DateCancel','CSA2010']]
vio.head(1)

start_date = '20'+year+'-01-01'
end_date = '20'+year+'-12-31'

mask = vio[ ( vio['DateNotice'] > start_date ) & ( vio['DateNotice'] <= end_date) ]
mask1 = mask[ ( pd.isnull( mask['DateAbate'] ) ) | ( mask['DateAbate'] >= end_date ) ]
mask2 = mask1[ pd.isnull( mask1['DateCancel'] ) | ( mask1['DateCancel'] > end_date ) ]
vio = mask2.copy()

vio.to_csv('vio_Filtered_Records.csv', index=False)

# Cell
# Aggregate Numeric Values by Sum
vio['vioCount'] = 1
vio = vio.groupby('CSA2010').sum(numeric_only=True)
vio = totalres[ ['CSA2010', 'totalres'+year] ].merge( vio, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Create the Indicator
vio['vio'] = vio['vioCount'] * 100 / totalres['totalres'+year]

# Create Baltimore's Record.
vio.at[55,'vioCount'] = vio['vioCount'].sum()
vio.at[55,'vio'] = vio['vioCount'].sum() * 100 / totalres['totalres'+year].sum()

vio.to_csv('35-violations'+year+'.csv', index=False)

vio.tail()

# Cell
# Aggregate Numeric Values by Sum
forclosure['foreCount'] = 1
fore = forclosure.groupby('CSA2010').sum(numeric_only=True)
# Make sure ALL csas and BaltimoreCity are included. among other things
fore = totalres[ ['CSA2010', 'totalres'+year] ].merge( fore, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Create the Indicator
fore['fore'] = fore['foreCount'] * 100 / fore['totalres'+year]

fore.at[55,'foreCount'] = fore['foreCount'].sum()
fore.at[55,'fore'] = fore['foreCount'].sum() * 100 / fore['totalres'+year].sum()

fore = fore[['CSA2010', 'foreCount', 'fore', 'totalres'+year ]]

fore.to_csv('33-fore'+year+'.csv', index=False)

fore.tail(60)

# Cell
resrehab = permitsCsa
resrehab['Field22'] = resrehab['typework']

use = ".SF.|.MF.|.DFAM.|.1-.|SF|MF|DFAM|1-.|.1-"
resrehab = resrehab[
  ( permitsCsa['existingus'].str.contains(use, regex=True, na=False) ) &
  ( permitsCsa['propuse'].str.contains(use, regex=True, na=False) ) &
  ( permitsCsa['casetype'].str.contains('.COM.|COM', regex=True, na=False) ) &
  ( permitsCsa['Field22'].str.contains('.AA.|.ADD.|.ALT.|AA|ADD|ALT|ADD', regex=True, na=False) ) &
  ( permitsCsa['cost'] >=5000 )
]

resrehab.to_csv('resrehab'+year+'_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', resrehab.size / len(resrehab.columns) )

# Aggregate Numeric Values by Sum
resrehab['resrehabCount'] = 1
resrehab = resrehab.groupby('CSA2010').sum(numeric_only=True)

# Make sure ALL csas and BaltimoreCity are included. among other things
resrehab = totalres[ ['CSA2010','totalres'+year] ].merge( resrehab, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Update the baltimore CSA.
resrehab.at[55,'resrehabCount'] = resrehab['resrehabCount'].sum()

# Create the Indicator
resrehab['resrehab'+year] = resrehab['resrehabCount'] * 100 / totalres['totalres'+year]

resrehab = resrehab[ ['CSA2010', 'resrehabCount', 'resrehab'+year, 'totalres'+year ] ]

resrehab.to_csv('36-resrehab'+year+'.csv', index=False)

resrehab.head()
resrehab.tail()

# Cell
demper = permitsCsa[
  ( permitsCsa['casetype'].str.contains('DEM|.DEM.|DEM.|.DEM', regex=True, na=False) )
]

filter = demper["PLANADDRES"] != ""
demper = demper[filter]


demper.to_csv('demper'+year+'_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', demper.size / len(demper.columns) )


# Aggregate Numeric Values by Sum
demper['demperCount'] = 1
demper = demper.groupby('CSA2010').sum(numeric_only=True)

# Make sure ALL csas and BaltimoreCity are included. among other things
demper = totalres[ ['CSA2010','totalres'+year] ].merge( demper, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Update the baltimore CSA.
demper.at[55,'demperCount'] = demper['demperCount'].sum()

# Create the Indicator
demper['demper'+year] = demper['demperCount'] * 100 / totalres['totalres'+year]

# Create the Indicator
demper['demper'+year] = demper['demperCount'] * 1000 / totalres['totalres'+year]

demper = demper[['CSA2010', 'demperCount', 'demper'+year, 'totalres'+year ]]

demper.to_csv('41-demper'+year+'.csv', index=False)

demper.head(60)

# Cell
# 2018 and 2017 is not working with the new datasets given (CSA LABELS)

use = "SF|MF|.SF.|.MF.|.SF|.MF|SF.|MF."
constper = permitsCsa
constper['Field22'] = constper['typework']
constper = constper[
  #(
  #    permitsCsa['existingus'].str.contains(use, regex=True, na=False) |
  #    permitsCsa['propuse'].str.contains(use, regex=True, na=False)
  #) &
  #( permitsCsa['casetype'].str.contains('COM|.COM.|COM.|.COM', regex=True, na=False) ) &
  ( constper['Field22'].str.contains('NEW|.NEW.|NEW.|.NEW', regex=True, na=False) )
]

constper = constper[constper["PLANADDRES"] != ""]

constper = constper[['CSA2010','existingus','propuse','casetype','Field22','PLANADDRES' ]]

constper.to_csv('constper'+year+'_Filtered_Records.csv', index=False)

print( 'Records Matching Query: ', constper.size / len(constper.columns) )

# Aggregate Numeric Values by Sum
constper['constperCount'] = 1
constper = constper.groupby('CSA2010').sum(numeric_only=True)

# Make sure ALL csas and BaltimoreCity are included. among other things
constper = totalres[ ['CSA2010','totalres'+year] ].merge( constper, left_on='CSA2010', right_on='CSA2010', how='outer' )

# Update the baltimore CSA.
constper.at[55,'constperCount'] = constper['constperCount'].sum()

# Create the Indicator
constper['42-constper'+year] = constper['constperCount'] * 1000 / totalres['totalres'+year]

constper.to_csv('42-constper'+year+'.csv', index=False)

constper.head(80)

# Cell
# sum( case when (lu like 'C' OR lu LIKE 'EC' OR lu LIKE 'I')

comprop = mdprop.copy()
comprop['comprop'+year] = 1
# mdprop = csa[['CSA','comprop19']]
comprop = comprop[ comprop['LU'].isin( ['C','EC','I'] ) ]
comprop = comprop.groupby('CSA2010').sum(numeric_only=True)
# Make sure ALL csas and BaltimoreCity are included. among other things
comprop = csa[ ['CSA2010','tpop10'] ].merge( comprop, left_on='CSA2010', right_on='CSA2010', how='outer' )
# Update the baltimore CSA.
comprop.at[55,'comprop'+year] = comprop['comprop'+year].sum()
comprop = comprop[['comprop'+year, 'CSA2010']]
comprop.head(58)

# Create the Indicator
comprop.to_csv('141-comprop'+year+'.csv', index=False)

# Cell
# get the permits file
crehab = permitsCsa.copy()

# Our Column to Sum on
crehab['crehab'+year] = 1
# Filter 1

print('No Filter:', crehab.shape[0])
print('Filter Cost:', crehab[crehab['cost'] >=5000].cost.shape[0])
print('Filter ExistingUse:', crehab.loc[crehab['existingus'].str.contains('2-|3-|4-|5-|6-|7-|COM|IND|BUS|AIR|ANIM|BAR|BEAU|DELI|FAC|ASM|ALV|DOTH|DWC|EDU|FOOD|HCF|HIH|HOS|MIXC|INS|MER|LIB|MNTL|MOB|PUB|STO|UT|VAC|VAL|DFAM') == True].shape[0])
# print('Filter Propuse:', crehab.loc[ crehab['propuse'].str.contains('COM|IND|BUS|AIR|ANIM|BAR|BEAU|DELI|FAC|ASM|ALV|DOTH|DWC|EDU|FOOD|HCF|HIH|HOS|MIXC|INS|MER|LIB|MNTL|MOB|PUB|STO|UT|DFAM') == True].shape[0])
print('Filter typework:', crehab[  crehab['typework'].isin( ['AA','ALT','ADD'] ) ].shape[0])

crehab.loc[ crehab['propuse'].str.contains('COM|IND|BUS|AIR|ANIM|BAR|BEAU|DELI|FAC|ASM|ALV|DOTH|DWC|EDU|FOOD|HCF|HIH|HOS|MIXC|INS|MER|LIB|MNTL|MOB|PUB|STO|UT|DFAM') == True].propuse.unique()

crehab = crehab.loc[crehab['existingus'].str.contains('2-|3-|4-|5-|6-|7-|COM|IND|BUS|AIR|ANIM|BAR|BEAU|DELI|FAC|ASM|ALV|DOTH|DWC|EDU|FOOD|HCF|HIH|HOS|MIXC|INS|MER|LIB|MNTL|MOB|PUB|STO|UT|VAC|VAL|DFAM') == True]
# crehab = crehab.loc[ crehab['propuse'].str.contains('COM|IND|BUS|AIR|ANIM|BAR|BEAU|DELI|FAC|ASM|ALV|DOTH|DWC|EDU|FOOD|HCF|HIH|HOS|MIXC|INS|MER|LIB|MNTL|MOB|PUB|STO|UT|DFAM') == True]
crehab = crehab[ crehab['cost'] >=5000 ]
crehab = crehab[  crehab['typework'].isin( ['AA','ALT','ADD'] ) ]
crehab.head(1)

# Cell
crehab = crehab.groupby('CSA2010').sum(numeric_only=True)
# Make sure ALL csas and BaltimoreCity are included. among other things
crehab = csa[ ['CSA2010','tpop10'] ].merge( crehab, left_on='CSA2010', right_on='CSA2010', how='outer' )
# Update the baltimore CSA.
crehab.at[55,'crehab'+year] = crehab['crehab'+year].sum()
crehab = crehab[['crehab'+year, 'CSA2010']]
crehab['crehab'+year] = crehab['crehab'+year] *100 / comprop['comprop'+year]

crehab.head(58)

# Create the Indicator
crehab.to_csv('142-crehab'+year+'.csv', index=False)