# Note: Script not accessible via __init__.py and is run directly by the developer and Github Actions
# updates swath_coverage_by_time_position.csv

# Built in Python functions
import logging
import os

# External Python libraries (installed via pip install)
from bs4 import BeautifulSoup
import pandas as pd
from urllib import request, error

# Internal Pydar reference to access functions, global variables, and error handling
import pydar

########################################################################

## Logging set up for .INFO
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
stream_handler = logging.StreamHandler()
logger.addHandler(stream_handler)

## FUNCTIONS TO WEB SCRAPE TO POPULATE swath_coverage_by_time_position.csv ################
def updateCsvSwathCoverage():
	# Update the csv script for swath_coverage_by_time_position.csv from the most recent JPL webpage
	# Retrieves information for each .LBL file that exists for CASSINI data files
	#		Estimated runtime: 15 minutes
	#		Returns: swath_coverage_by_time_position.csv in data/ folder

	logger.info("Refreshing: swath_coverage_by_time_position.csv")

	# Get all Titan Flybys with most up to date versions
	coradr_ids = []
	ids_with_no_bidr = []
	coradr_csv_file = os.path.join(os.path.dirname(__file__), 'data', 'coradr_jpl_options.csv')  # get file's directory, up one level, /data/*.csv
	coradr_dataframe = pd.read_csv(coradr_csv_file)
	for index, row in coradr_dataframe.iterrows():
		row = row.tolist()
		if row[1] == True:
			if row[4] == False: # if is a Titan flyby but "Contains BIDR" is False
				ids_with_no_bidr.append(row[0])
			if coradr_ids != []:
				if coradr_ids[-1].split("_V")[0] in row[0]: # replace the older version with the most recent version
					coradr_ids[-1] = row[0]
				else:
					coradr_ids.append(row[0])
			else:
				coradr_ids.append(row[0])

	data_type_dict = {"F":"Primary Dataset (Linear Scale)", 
					"B": "Primary Dataset in Unsigned Byte Format (Normalized dB)",
					"S": "Normalized SAR (Physical Scale) with Thermal/Quantized Noise Removed",
					"X": "Noise for SAR without Incidence Angle Correction (Physical Scale)",
					"U":"Sigma0 without Incidence Angle (Linear Scale)", 
					"D": "Subtracted STD SAR",
					"E":"Incidence Angle Map", 
					"T":"Latitude Map",
					"N":"Longitude Map",
					"M":"Beam Mask Map",
					"L":"Number of Looks Map"}

	resolution_dict = {"B": 2, "D": 8, "F": 32, "G": 64, "H": 128, "I": 256} #  pixels/degree

	# Retrieve a list of all the .LBL for each CORADR ID (different for each resolution)
	lbl_information = []
	for radar_id in coradr_ids:
		if radar_id not in ids_with_no_bidr:
			base_url = "https://pds-imaging.jpl.nasa.gov/data/cassini/cassini_orbiter/{0}/DATA/BIDR/".format(radar_id)
			base_html = request.urlopen(base_url).read()
			soup = BeautifulSoup(base_html, 'html.parser')
			table = soup.find('table', {"id": "indexlist"})
			table_text = (table.text).split("\n")
			url_filenames = []
			all_bidr_files = []
			for txt in table_text:
				if txt.startswith('BI'):
					filename = (txt.split('/')[0]).split(".")[0]
					if 'LBL' in (txt.split('/')[0]).split(".")[1]:
						lbl = []
						filename += '.LBL'
						bidr_url = "https://pds-imaging.jpl.nasa.gov/data/cassini/cassini_orbiter/{0}/DATA/BIDR/{1}".format(radar_id, filename)
						logger.info("Retrieving LBL information: {0}".format(bidr_url))
						lbl = [radar_id, None, None, None, None, None, None, None, None, None, None, None, None, None]
						lbl[1] = pydar.convertObservationNumberToFlybyID(radar_id.split("_")[1])
						lbl[2] = (filename.split("_")[2]).split("S")[1] # Segment Number
						lbl[3] = filename
						lbl[4] = filename[2] # Data Type
						lbl[5] = data_type_dict[filename[2]] # Data Type, with full title
						lbl[6] = resolution_dict[filename[4]] # Resolution
						with request.urlopen(bidr_url) as lbl_file:
							for line in (lbl_file.read().decode("UTF-8")).split("\n"):
								if "TARGET_NAME" in line:
									lbl[7] = line.split("=")[1].strip()
								if "MAXIMUM_LATITUDE" in line:
									max_lat = line.split("=")[1].strip()
									max_lat = max_lat.split("<")[0] 
									lbl[8] = max_lat
								if "MINIMUM_LATITUDE" in line:
									min_lat = line.split("=")[1].strip()
									min_lat = min_lat.split("<")[0] 
									lbl[9] = min_lat
								if "EASTERNMOST_LONGITUDE" in line:
									east_long = line.split("=")[1].strip()
									east_long = east_long.split("<")[0] 
									lbl[10] = east_long
								if "WESTERNMOST_LONGITUDE" in line:
									west_long = line.split("=")[1].strip()
									west_long = west_long.split("<")[0] 
									lbl[11] = west_long
								if "START_TIME" in line:
									lbl[12] = line.split("=")[1].strip()
								if "STOP_TIME" in line:
									lbl[13] = line.split("=")[1].strip()
							lbl_information.append(lbl)

	# Wrte to CSV
	header_options = ["CORADR ID",
					"FLYBY ID",
					"SEGMENT NUMBER",
					"FILENAME",
					"DATE TYPE SYMBOL",
					"DATE TYPE",
					"RESOLUTION (pixels/degrees)",
					"TARGET_NAME",
					"MAXIMUM_LATITUDE (Degrees)",
					"MINIMUM_LATITUDE (Degrees)",
					"EASTERNMOST_LONGITUDE (Degrees)",
					"WESTERNMOST_LONGITUDE (Degrees)",
					"START_TIME",
					"STOP_TIME"
					]
	df = pd.DataFrame(lbl_information, columns=header_options)
	df = df.sort_values(by=["CORADR ID"])
	df.to_csv(os.path.join(os.path.dirname(__file__), 'data', 'swath_coverage_by_time_position.csv'), header=header_options, index=False)

if __name__ == '__main__':
	updateCsvSwathCoverage() # 		updates swath_coverage_by_time_position.csv
