#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jan 17 10:54:49 2023

@author: psakicki
"""

import datetime as dt
import subprocess
import urllib
from geodezyx import utils, conv, operational
from xml.etree import ElementTree as et
import os
from threading import Thread
import time


#### Import the logger
import logging
log = logging.getLogger(__name__)

def log_subprocess_output(pipe,logger=None,file=None,file2=None):
    """
    Intern fuction for subprocess_frontend2
    to write the stdout/err in the console logger + a logfile
    """
    for line in iter(pipe.readline, b''): # b'\n'-separated lines
        line_clean = line.decode().strip()
        if file:
            file.write(line_clean + "\n") 
        if file2:
            file2.write(line_clean + "\n") 
        if logger:
            logger(line_clean)
    return 
    
    
    
def subprocess_frontend2(cmd_in,
                         save_log=True,
                         log_dir=None,
                         log_name_out="out.log",
                         log_name_err="err.log",
                         logname_timestamp=True,
                         err_also_in_outfile=True,
                         logger_objt_level_out=None,
                         logger_objt_level_err=None):
    """
    A generic frontend to run an extern command through subprocess
    and write the stdout and stderr outputs in log files.
    
    Parameters
    ----------
    cmd_in : str
        The subprocess command.
    save_log : str, optional
        export as log the stdout/stderr in files. The default is True.
    log_dir : str, optional
        directory where the logs will be stored. The default is None.
    log_name_out : str, optional
        filename of the stdout log. The default is "out.log".
    log_name_err : str, optional
        filename of the stderr log. The default is "err.log".
    logname_timestamp : str, optional
        add a timestamp as prefix. The default is True.
    logger_objt_level_out : method of a Logger object, optional
        set the logger level of the stdout messages.
        can be logger.info or logger.debug for instance.
        The default is None (logger.info per default then).
    logger_objt_level_err : method of a Logger object, optional
        set the logger level of the stderr messages.
        can be logger.error or logger.critical for instance.
        The default is None (logger.error per default then).
        
    Returns
    -------
    exitcode : int
        the exit code of the command.
        
    Notes
    -----
    Inspired by:
    https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
    And for the threading:
    https://stackoverflow.com/questions/6809590/merging-a-python-scripts-subprocess-stdout-and-stderr-while-keeping-them-disti
    """
    
    from subprocess import Popen, PIPE, STDOUT
    now = utils.get_timestamp()
    
    #### manage the paths of the output logs
    if save_log:
        if not log_dir:
            log_dir = os.getcwd()
    
        if logname_timestamp:
            prefix = now + "_"
        else:
            prefix = ""
    
        err_file = open(log_dir + "/" + prefix + log_name_err, "w+")
        out_file = open(log_dir + "/" + prefix + log_name_out, "w+")
        
        if err_also_in_outfile:
            err_in_outfile = out_file
        else:
            err_in_outfile = None
            
    else:
        out_file = None
        err_file = None
        err_in_outfile = None
        
    ## set logger level
    if logger_objt_level_out:
        loggerout = logger_objt_level_out
    else:
        loggerout = log.info
        
    if logger_objt_level_err:
        loggererr = logger_objt_level_err
    else:
        loggererr = log.error
        
                                
    ## arguments must be splitted 
    if type(cmd_in) is str:
        cmd_split = cmd_in.split()
    else:
        cmd_split = cmd_in
    
    ####### Run the command here  #####################
    process = Popen(cmd_split,
                    stdout=PIPE,
                    stderr=PIPE)
    ##################################################
    
    ### Get out/err simultaneously
    stdout_thread = Thread(target=log_subprocess_output,
                           args=(process.stdout,
                                 loggerout,
                                 out_file))
    stderr_thread = Thread(target=log_subprocess_output,
                           args=(process.stderr,
                                 loggererr,
                                 err_file,
                                 err_in_outfile))
    
    stderr_thread.start()
    stdout_thread.start()

    while stdout_thread.is_alive() or stderr_thread.is_alive():
        pass ### do nothing while the threads are runing

    exitcode = process.wait() # 0 means success

    if save_log:
        out_file.close()
        err_file.close()       
    
    return exitcode


def groops_basic_runner(xml_cfg_path="",
                        global_var_dict=dict(),
                        xml_var_dict=dict(),
                        dry_run=False,
                        verbose=False,
                        log_dir=None,
                        groops_bin_path='/opt/softs_gnss/groops/bin/groops'):
    """
    

    Parameters
    ----------
    xml_cfg_path : str, optional
        the XML config file for GROOPS. The default is "".
    global_var_dict : dict, optional
        A dictionnary to change the global variables values, like:
        global_var_dict["global_var_name"] = new_value
        The default is dict().
    xml_var_dict : dict, optional
        A dictionnary to change the values in the config XML file.
        Not implemented yet. 
        The default is dict().
    dry_run : bool, optional
        If True print the command but do not run it . The default is False.
    verbose : bool, optional
        print the detailled GROOPS's output. The default is False.
    log_dir : str, optional
        If provided, directory where the logs are stored. The default is None.
    groops_bin_path : TYPE, optional
        Path of the GROOPS bin. 
        The default is '/opt/softs_gnss/groops/bin/groops'.

    Returns
    -------
    None.

    """


    global_args_str = ""
    for key, val in global_var_dict.items():
        global_args_str = global_args_str + " ".join((" --global",key+"="+str(val)))
        
        
    command = " ".join((groops_bin_path,global_args_str,xml_cfg_path))
    log.info("groops command:")
    log.info("%s",command)
    
    xml_cfg_bn = os.path.basename(xml_cfg_path)

    if verbose:
        log.setLevel(logging.DEBUG)
    else:
        log.setLevel(logging.INFO)

    if not log_dir:
        save_log = False
    else:
        save_log = True


    if not dry_run:
        subprocess_frontend2(command,
                             save_log=True,
                             log_dir=log_dir,
                             log_name_out= xml_cfg_bn + ".out.log",
                             log_name_err= xml_cfg_bn + ".err.log",
                             logname_timestamp=True,
                             logger_objt_level_out=log.debug)
    
    log.setLevel(logging.INFO)
    return 


def vmf_tropo_downloader(output_dir,
                         startdate = dt.datetime(2019,1,1),
                         enddate = dt.datetime(2019,1,1),
                         model = "VMF3",
                         version = "OP"):
    
    Dates_range = conv.dt_range(startdate,enddate)
    
    url_root = "https://vmf.geo.tuwien.ac.at"
    dir_root = output_dir
    subdir   = "trop_products/GRID/1x1/"
    
    Files_hour_list = []
    for currdate in Dates_range:
        subdir_date = os.path.join(subdir,
                                   model,
                                   model + "_" +  version,
                                   str(currdate.year))
        
        
        url_date = os.path.join(url_root,subdir_date)
        dir_date = os.path.join(dir_root,subdir_date)
        utils.create_dir(dir_date)

        date_str = conv.dt2str(currdate,"%Y%m%d") 
        
        for hourext in ("H00","H06","H12","H18"):      
            filename_hour = model + "_" + date_str + "." + hourext
            url_hour = os.path.join(url_date,filename_hour)
            outfile_hour = os.path.join(dir_date,filename_hour)
            
            log.info("download %s to %s",url_hour,outfile_hour)
            
            Files_hour_list.append(outfile_hour)
            if not utils.empty_file_check(outfile_hour):
                log.info("%s exists, download skipped",outfile_hour)
                continue
            else:
                DownObjt = urllib.request.urlretrieve(url_hour, 
                                                      outfile_hour)   
    return Files_hour_list
    
#%%

###############################################################################
###############################################################################

def groops_ppp_full_runner(rinex_path,
                           project_name,
                           igs_ac_10char,
                           cfg_files_dict,
                           log_root_dir,
                           vmf_tropo_root_dir,
                           prods_gnss_root_dir,
                           cfg_files_root_dir):
    
    ###############################################################################
    ######## Set python fct variables
    
    
    #### Internal debug variables
    log.setLevel(logging.INFO)
    debug_sleep_time = 2
    dry_run=False


    prods_gnss_dir = os.path.join(prods_gnss_root_dir,igs_ac_10char)
    
    #### Determine if products are MGEX or not
    if igs_ac_10char[4:7] == "MGX":
        mgex=True
    else:
        mgex=False
    #### Determine if products are repro or not    
    if (igs_ac_10char[4] == "R") and (igs_ac_10char[6] == "3"):
        repro=3
        mgex=False        
    else:
        repro=0
    
    #### Define project name 
    project_name_use = project_name + "_" + igs_ac_10char 

    #### Define rinex-linked variables
    site4char = os.path.basename(rinex_path)[:4]
    date_rnx = conv.rinexname2dt(rinex_path)
    date_rnx_mjd = int(conv.dt2MJD(date_rnx))
    date_rin_ymd = conv.dt2str(date_rnx,"%Y-%m-%d")
    date_rin_doy = utils.join_improved("-",*list(reversed(conv.dt2doy_year(date_rnx))))
    date_rin_dow = utils.join_improved("-",*conv.dt2gpstime(date_rnx))
    
    #### Create associated log directory    
    log_dir = os.path.join(log_root_dir,"_".join((utils.get_timestamp(),
                                                  project_name_use,
                                                  site4char,
                                                  date_rin_ymd)))
    utils.create_dir(log_dir)
    

    ###############################################################################
    ######## Initial print
    log.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
    log.info("****** GROOPS RUNNER Start ****************************************")
    log.info("** RINEX: %s", os.path.basename(rinex_path))
    log.info("** AC: %s", igs_ac_10char)
    log.info("** date: %s, MJD: %s",
             date_rin_ymd,
             date_rnx_mjd)
    log.info("** year-doy: %s, GPS week-dow: %s",
             date_rin_doy,
             date_rin_dow)
    
    ###############################################################################
    ##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    ######## VMF TROPO
    
    ###############################################################################
    ######## Download Tropo
    log.info("****** VMF troposphere grids download *****************************")
    
    VMF_files  = vmf_tropo_downloader(vmf_tropo_root_dir,
                                      startdate = date_rnx  - dt.timedelta(days=0),
                                      enddate   = date_rnx)
    
    ###############################################################################
    ######## Convert Tropo
    log.info("****** VMF troposphere grids conversion ***************************")
    
    xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convTropo"]
    
    global_var_dict = dict()
    global_var_dict["timeStart"] = date_rnx_mjd
    global_var_dict["timeEnd"]   = date_rnx_mjd
    global_var_dict["groopsInpVmfGridFile1"] = VMF_files[-4]
    global_var_dict["groopsInpVmfGridFile2"] = VMF_files[-3]
    global_var_dict["groopsInpVmfGridFile3"] = VMF_files[-2]
    global_var_dict["groopsInpVmfGridFile4"] = VMF_files[-1]
    
    groops_basic_runner(xml_cfg_path,
                        global_var_dict,
                        log_dir=log_dir,
                        dry_run=dry_run)
    
    ###############################################################################
    ##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    ######## PRODUCTS
    
    ###############################################################################
    ######## Check if converted products exists
    log.info("****** Converted GNSS products existance check ********************")
    
    #### HARDCODED !!!!! XXXXXXXXXXXXXXXXX
    conv_prod_root_dir = "/home/ovsgnss/020_CALC/groops_process/021_conv_igs_prods"
    conv_prod_dir = os.path.join(conv_prod_root_dir,igs_ac_10char,date_rin_ymd)
    
    All_conv_prods = utils.find_recursive(conv_prod_dir,"*dat")
    Clk_conv_prods = utils.find_recursive(conv_prod_dir,"clock*dat")
    Orb_conv_prods = utils.find_recursive(conv_prod_dir,"orbit*dat")
        
    if len(Orb_conv_prods) > 12: ## 12 is arbitrary
        download_prods = False
        log.info("Products download/conversion skipped, %s converted files found in %s",
                 len(All_conv_prods),conv_prod_dir)
    else:
        download_prods = True
        log.info("Products download/conversion will start, no files found in %s",conv_prod_dir)
    
    ###############################################################################
    ######## Download Products
    
    if download_prods:
        log.info("****** GNSS products download *********************************")
    
        download_fct = operational.multi_downloader_orbs_clks_2
        
        Prod_types  = ["sp3","clk","bia","obx"]
        AC_names    = [igs_ac_10char]
        Prods_out_list = download_fct(prods_gnss_dir,
                                      date_rnx,
                                      date_rnx,
                                      AC_names,
                                      Prod_types,
                                      archtype ='week',
                                      archive_center="cddis",
                                      parallel_download=1,
                                      repro=repro,
                                      mgex=mgex)
        
        if not Prods_out_list:
            log.warning("No downloaded products found locally, abort")  
            raise Exception
        
        ###### Check the downloaded products (IGS AC ID)
        Igs_ac_10char_list_out = [os.path.basename(e)[:10] for e in Prods_out_list]
        Igs_ac_10char_list_out = list(set(Igs_ac_10char_list_out))
            
        if len(Igs_ac_10char_list_out) > 1:
            log.warning("Several downloaded product types found: %s",
                        Igs_ac_10char_list_out)
               
        if  Prods_out_list and (igs_ac_10char != Igs_ac_10char_list_out[0]):
            log.warning("Product type downloaded != requested: %s",
                        Igs_ac_10char_list_out[0],
                        igs_ac_10char)
    
    ###############################################################################
    ######## Convert Products: clocks
    if download_prods:
        log.info("****** GNSS products conversion: clocks *******************")
    
        xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convProds_clock"] 
        
        global_var_dict = dict()
        global_var_dict["timeStart"] = date_rnx_mjd
        global_var_dict["timeEnd"] = date_rnx_mjd + 1
        global_var_dict["igsAC10Char"] = igs_ac_10char
        global_var_dict["inpIgsProdsDir"] = prods_gnss_dir
        groops_basic_runner(xml_cfg_path,
                            global_var_dict,
                            log_dir=log_dir,
                            dry_run=dry_run)

    
        time.sleep(debug_sleep_time)    
    
    ###############################################################################
    ######## Convert Products: bias
    if download_prods:
        log.info("****** GNSS products conversion: bias *******************")
    
        xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convProds_bias"]
        
        groops_basic_runner(xml_cfg_path,
                            global_var_dict,
                            log_dir=log_dir,
                            dry_run=dry_run)

        time.sleep(debug_sleep_time)    
    
    ###############################################################################
    ######## Convert Products: orbits
    if download_prods:
        log.info("****** GNSS products conversion: orbits *******************")
    
        xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convProds_orbit"]
        
        groops_basic_runner(xml_cfg_path,
                            global_var_dict,
                            log_dir=log_dir,
                            dry_run=dry_run)

        time.sleep(debug_sleep_time)    

    ###############################################################################
    ######## Convert Products: alternative attitude
    if download_prods:
        log.info("****** GNSS products conversion: alternative attitude *****")
    
        xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convProds_alt_attitude"]
        
        groops_basic_runner(xml_cfg_path,
                            global_var_dict,
                            log_dir=log_dir,
                            dry_run=dry_run)

        time.sleep(debug_sleep_time)    



    ###############################################################################
    ######## Convert Products: attitude
    if download_prods:
        log.info("****** GNSS products conversion: attitude *****")
    
        xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convProds_attitude"]
        
        groops_basic_runner(xml_cfg_path,
                            global_var_dict,
                            log_dir=log_dir,
                            dry_run=dry_run)

        time.sleep(debug_sleep_time)    
    
    ###############################################################################
    ##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    ######## RINEX & STATION INFO
    
    ###############################################################################
    ######## convert sitelog > station info
    
    ## cfg_files_dict["convSite_sitelog"] 
    
    # log.info("\n****** sitelog > station info conversion **************************")
    
    # xmltree = et.parse(xml_cfg_generic)
    # # get input files
    # xmltree.find('.//rinexo').text
    
    # xml_cfg_path=cfg_files_root_dir + "022_groopsConvert_GnssObs_v02b.xml"
    
    # global_var_dict = dict()
    # global_var_dict["timeStart"] = date_rnx_mjd
    # global_var_dict["timeEnd"]   = date_rnx_mjd + 1
    # global_var_dict["site4Char"] = site4char
    # global_var_dict["inpGnssObsRnxFile"] = rinex_path
    
    # groops_basic_runner(xml_cfg_path,
    #                     global_var_dict)
    
    ###############################################################################
    ######## Convert RINEX
    
    log.info("****** RINEX observation conversion *******************************")
    
    xml_cfg_path=cfg_files_root_dir + cfg_files_dict["convSite_rnxObs"]
    
    global_var_dict = dict()
    global_var_dict["timeStart"] = date_rnx_mjd
    global_var_dict["timeEnd"]   = date_rnx_mjd + 1
    global_var_dict["site4Char"] = site4char
    global_var_dict["inpGnssObsRnxFile"] = rinex_path
    
    groops_basic_runner(xml_cfg_path,
                        global_var_dict,
                        log_dir=log_dir,
                        dry_run=dry_run)

    
    ###############################################################################
    ######## Edit station list
    
    log.info("****** Station list edition ***************************************")
    
    #### HARDCODED !!!!! XXXXXXXXXXXXXXXXX
    station_list_path='/opt/softs_gnss/groops/stationlists/station_list_OPERA_01a.txt'
    F=open(station_list_path,"w+")
    F.write(site4char)
    F.close()
    
    
    ###############################################################################
    ##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
    ######## Run Processing
    
    log.info("****** Processing run *********************************************")
    
    xml_cfg_path=cfg_files_root_dir + cfg_files_dict["gnssProcessing"] 
    
    global_var_dict = dict()
    global_var_dict["igsAC10Char"] = igs_ac_10char
    global_var_dict["projectName"] = project_name_use
    global_var_dict["timeStart"] = date_rnx_mjd
    global_var_dict["timeEnd"]   = date_rnx_mjd + 1
    
    groops_basic_runner(xml_cfg_path,
                        global_var_dict,
                        log_dir=log_dir,
                        dry_run=dry_run)

    
    log.info("****** GROOPS RUNNER End ******************************************")
    log.info("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%")
    
    return

### ===========================================================================
### ===========================================================================


