#!/usr/bin/python
# -*- coding: utf-8 -*-
# Filename: CMADaasAccess.py

'''
Created on Sep 15, 2020

@author: anduin
'''
import meteva_base
import datetime
import json
from meteva_base.io.loglib import *
from meteva_base.io.httpconn import HttpConn
from meteva_base.io.SignGenUtil import SignGenUtil
import numpy as np

class CMADaasAccess():
    """description of class
    cmadaas data access
    """
    @staticmethod
    def get_times(dts):
        try:
            if type(dts) is list:
                return ','.join(dt.strftime('%Y%m%d%H%M%S') for dt in dts)
            else:
                return dts.strftime('%Y%m%d%H%M%S')
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess get_times except %s %s' % (str(dts), str(data)))
            raise data
            # return None

    @staticmethod
    def set_times_in_params(dts, params, deep_copy=True):
        try:
            rst = params
            if deep_copy:
                import copy
                rst = copy.deepcopy(params)

            rst['times'] = CMADaasAccess.get_times(dts)

            return rst
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess set_times_in_params except %s %s' % (str(dts), str(data)))
            raise data
            # return None

    @staticmethod
    def get_timerange(dts):
        try:
            rst = ''
            if type(dts) is list:
                rst = ','.join(dt.strftime('%Y%m%d%H%M%S') for dt in dts)
            else:
                rst = dts.strftime('%Y%m%d%H%M%S')

            return '[' + rst + ']'
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess get_timerange except %s %s' % (str(dts), str(data)))
            raise data
            # return None

    @staticmethod
    def set_timerange_in_params(dts, params, deep_copy=True):
        try:
            rst = params
            if deep_copy:
                import copy
                rst = copy.deepcopy(params)

            rst['timeRange'] = CMADaasAccess.get_timerange(dts)

            return rst
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess set_timerange_in_params except %s %s' % (str(dts), str(data)))
            raise data
            # return None

    @staticmethod
    def get_query(params):
        '''
        生成url中query部分
        '''
        try:
            querystr = ''
            for key, item in params.items():
                querystr += key + "=" + str(item) + "&"

            if (querystr):
                querystr = querystr[:-1]

            return querystr
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess get_query except %s %s' % (str(params), str(data)))
            raise data
            # return None

    @staticmethod
    def get_url(url, qparams):
        '''
        生成url
        '''
        # import copy
        try:
            # signParams = copy.deepcopy(qparams)
            signParams = qparams
            signString = SignGenUtil.getSign(signParams)
            if signString is None:
                LogLib.error('CMADaasAccess getUrl getSign error %s %s' % (url, str(qparams)))
                return None

            signParams['sign'] = signString
            signParams.pop('pwd')

            querystr = CMADaasAccess.get_query(signParams)

            return url + querystr
        except Exception as data:
            # print(str(data))
            LogLib.error('CMADaasAccess getUrl except %s %s %s' % (url, str(qparams), str(data)))
            raise data
            # return None

    @staticmethod
    def get_default_headers():
        return {'Accept': 'application/json, text/plain, */*',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}

    # only json is supported
    # 从cmadaas获得json格式数据，使用json库解析，构建一个dataframe返回。
    @staticmethod
    def read_data_from_cmadaas(fullurl):
        try:
            LogLib.info('CMADaasAccess read_data_from_cmadaas start %s' % (str(fullurl)))
            headers = CMADaasAccess.get_default_headers()
            host, url, ishttp = HttpConn.parseUrl(fullurl)
            rststatus, rstcontent, rstheaders = HttpConn.getWebPage(host, url, headers=headers, ishttp=ishttp)
            if rststatus != 200:
                LogLib.error('CMADaasAccess read_data_from_cmadaas net error %d %s' % (rststatus, str(fullurl)))
                return None

            jsonobj = json.loads(rstcontent)
            if jsonobj is None:
                LogLib.error('CMADaasAccess read_data_from_cmadaas json loads error %s %s' % (rstcontent, str(fullurl)))
                return None

            if 'returnCode' not in jsonobj:
                LogLib.error(
                    'CMADaasAccess read_data_from_cmadaas json no returnCode %s %s' % (rstcontent, str(fullurl)))
                return None

            if jsonobj['returnCode'] != '0':
                if jsonobj['returnCode'] == '-1':
                    LogLib.info('CMADaasAccess read_data_from_cmadaas json no data %s %s' % (rstcontent, str(fullurl)))
                    return []
                else:
                    LogLib.error('CMADaasAccess read_data_from_cmadaas json no returnCode or returnCode != 0 %s %s' % (
                    rstcontent, str(fullurl)))
                    return None

            return jsonobj['DS']
        except Exception as data:
            LogLib.error('CMADaasAccess read_data_from_cmadaas except %s %s' % (str(fullurl), str(data)))
            return None

    @staticmethod
    def read_griddata_from_cmadaas(fullurl, time=None, dtime=None):
        try:
            LogLib.info('CMADaasAccess read_data_from_cmadaas start %s' % (str(fullurl)))
            headers = CMADaasAccess.get_default_headers()
            host, url, ishttp = HttpConn.parseUrl(fullurl)
            rststatus, rstcontent, rstheaders = HttpConn.getWebPage(host, url, headers=headers, ishttp=ishttp)
            if rststatus != 200:
                LogLib.error('CMADaasAccess read_data_from_cmadaas net error %d %s' % (rststatus, str(fullurl)))
                return None

            jsonobj = json.loads(rstcontent)
            if jsonobj is None:
                LogLib.error('CMADaasAccess read_data_from_cmadaas json loads error %s %s' % (rstcontent, str(fullurl)))
                return None

            if 'returnCode' not in jsonobj:
                LogLib.error(
                    'CMADaasAccess read_data_from_cmadaas json no returnCode %s %s' % (rstcontent, str(fullurl)))
                return None

            griddata = CMADaasAccess.griddata_from_cmadaas_json(jsonobj, time=time, dtime=dtime)
            return griddata
        except Exception as data:
            LogLib.error('CMADaasAccess read_data_from_cmadaas except %s %s' % (str(fullurl), str(data)))
            return None

    @staticmethod
    def griddata_from_cmadaas_json(json, time=None, dtime=None):
        try:
            lonS = round(json['startLon'], 4)
            # lonE = round(json['endLon'],3)
            latS = round(json['startLat'], 4)
            # latE = round(json['endLat'],3)
            dlon = round(json['lonStep'], 4)
            dlat = round(json['latStep'], 4)
            nlon = int(json['lonCount'])
            nlat = int(json['latCount'])
            lonE = lonS + (nlon - 1) * dlon
            latE = latS + (nlat - 1) * dlat
            # print(lonS,dlon,nlon,lonE)
            data = np.array(json['DS'])

            # data
            # print(data.shape)

            if time is not None:
                if not isinstance(time, list):
                    time = [time]  # 确保time为列表
            if dtime is not None:
                if not isinstance(dtime, list):
                    dtime = [dtime]  # 确保dtime为列表
            # print(time,dtime)
            grid_info = meteva_base.grid([lonS, lonE, dlon], [latS, latE, dlat]
                                         , gtime=time
                                         , dtime_list=dtime)
            # print(grid_info)
            grid = meteva_base.grid_data(grid_info, data)
        except Exception as data:
            print(json)
            return None
        return grid

    @staticmethod
    def dataframe_from_dict_json(j_dict, default=np.nan):
        import pandas as pd
        try:
            LogLib.info('CMADaasAccess Convert dataFrame from JSON_dict {0}'.format(
                datetime.datetime.now().strftime("%Y%m%d%H%M")))
            pddata = pd.DataFrame.from_dict(j_dict)
            ## 更改dataframe
            sta = meteva_base.sta_data(pddata, columns=['time', 'id', 'lat', 'lon', 'data0'])
            sta.level = 0
            sta.dtime = 0
            sta.time = pd.to_datetime(sta.time)
            sta.level = sta.level.astype(np.int16)
            sta.id = sta.id.astype(np.int32)
            sta.dtime = sta.dtime.astype(np.int16)
            sta.lon = sta.lon.astype(np.float32)
            sta.lat = sta.lat.astype(np.float32)
            try:
                sta.data0 = sta.data0.astype(np.float32)
                sta.data0[sta.data0 >= 990000] = default
            except:
                pass
            return (sta)
        except Exception as data:
            LogLib.error('CMADaasAccess Convert dataFrame from JSON_dict {0}'.format(
                datetime.datetime.now().strftime("%Y%m%d%H%M")))
            return (None)

    # 从cmadaas获得文件数据。
    @staticmethod
    def read_file_from_cmadaas(fullurl):
        try:
            LogLib.info('CMADaasAccess read_file_from_cmadaas start %s' % (str(fullurl)))
            headers = CMADaasAccess.get_default_headers()
            host, url, ishttp = HttpConn.parseUrl(fullurl)
            rststatus, rstcontent, rstheaders = HttpConn.getWebPage(host, url, headers=headers, ishttp=ishttp)
            if rststatus != 200:
                LogLib.error('CMADaasAccess read_file_from_cmadaas net error %d %s' % (rststatus, str(fullurl)))
                return None
            return (rstcontent, rstheaders)
        except Exception as data:
            LogLib.error('CMADaasAccess read_file_from_cmadaas except %s %s' % (str(fullurl), str(data)))
            return None

    @staticmethod
    def write_data_to_file(save_path, grd, creat_dir=True):
        import os
        try:
            LogLib.info('FixWriteData write_data_to_file start %s' % (save_path))
            if creat_dir:
                path, file = os.path.split(save_path)
                if not os.path.exists(path):
                    os.makedirs(path)
            with open(save_path, 'wb') as f:
                f.write(grd)
            LogLib.info('FixWriteData write_data_to_file over %s' % (save_path))
            return True
        except Exception as data:
            LogLib.error('FixWriteData write_data_to_file except %s' % (str(save_path)))
            raise data

    ## 生成datetime类
    def get_datetime_from_str(dt='202011110200'):  # dt可以为任意长度字符串，也可为datetime类
        if type(dt) == str:
            # 用户输入2019041910十位字符，后面补全加0000，为14位统一处理
            if len(dt) == 4:
                num1 = dt + "0101000000"
            elif len(dt) == 6:
                num1 = dt + "01000000"
            elif len(dt) == 8:
                num1 = dt + "000000"
            elif len(dt) == 10:
                num1 = dt + "0000"
            elif len(dt) == 12:
                num1 = dt + "00"
            elif len(dt) == 14:
                num1 = dt
            else:
                print("输入日期有误，请检查！")
            # 统一将日期变为datetime类型
            time = datetime.datetime.strptime(num1, '%Y%m%d%H%M%S')
        else:
            time = dt
        return (time)

    ## 根据参数字典，生成REST格式url。
    @staticmethod
    def combine_url_from_para(qprams, time=datetime.datetime(2020, 11, 27, 0, 0), url=None,
                              SN_id="NMIC_MUSIC_CMADAAS", dataFormat='json', userId=None, pwd=None,
                              defaultNan=9999, show_url=False, time_name='times'):
        """
        dt： 可以为字符串str，如“202011110200”；也可以为datetime类时间数据
        userID: 业务账号，必须填
        pwd： 业务账户密码，必须填
        dataFormat: 返回数据格式，默认为json
        show_url:  是否完整显示url，默认为不显示
        url: cmadaas的url前面固定部分，默认'http://10.40.17.54/music-ws/api?'
        SN_id: cmadaas服务节点代码，默认'NMIC_MUSIC_CMADAAS'
        time_name:时间参数的字典名(time或者times)
        """
        import copy
        params = copy.copy(qprams)
        LogLib.info('OBS Download from CmaDaas: {0}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M")))

        ## 账户处理
        if userId is None:
            userId = meteva_base.cmadaas_set[2]
            LogLib.warning(
                'NO userID input,Using default USER: {0}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M")))
        if pwd is None:
            pwd = meteva_base.cmadaas_set[3]
            LogLib.warning(
                'NO Password input,Using default pwd: {0}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M")))
        ## 添加字典变量要素
        if "timeRange" in params:
            params['timeRange'] = (str(params['timeRange']).replace(' ', ''))
        if "userId" not in params:
            params["userId"] = userId
        #            params = params.pop("userId")
        if "pwd" not in params:
            params["pwd"] = pwd
        if not (("times" in params) or ("time" in params) or ("timeRange" in params)):
            params[time_name] = CMADaasAccess.get_times(time)
        if "serviceNodeId" not in params: params["serviceNodeId"] = SN_id
        if "timestamp" not in params: params["timestamp"] = str(int(datetime.datetime.now().timestamp() * 1000))
        if "nonce" not in params: params["nonce"] = '3696663f-d202-4570-b39d-16306f419575'
        if "dataFormat" not in params: params['dataFormat'] = dataFormat
        ## 生成url

        # print(params['timeRange'])
        if url is None:
            url = "http://" + meteva_base.cmadaas_set[0] + "/music-ws/api?"

        urlstr = CMADaasAccess.get_url(url, params)
        if show_url is True: print(urlstr)
        ## 返回url
        return (urlstr)

    @staticmethod
    def get_obs_micaps3_from_cmadaas(qparams, time="202011110200", save_path=None, filename=None, defaultNan=9999,
                                     show=False,
                                     **kargs):
        """
        函数返回：meteva形式的pandas.DataFrame格式；如果申明sava_path变量，可直接输出Micaps3文件
        ###
        qparams: 包含数据、列表、下载要素的字典容器
        time： 可以为字符串str，如“202011110200”；也可以为datetime类时间数据
        userID: 业务账号，必须填
        pwd： 业务账户密码，必须填
        save_path: 存储文件目录，若不填写则不输出
        filename:  存储文件名,可以使用通配符(如YYYYMMDDHHFF.TTT即表示“年(四位)月(两位)日(两位)时(两位)分(两位).时效(三位)”；缺省-YYYYMMDDHHFF.000.m3)
        show_url:  是否完整显示url，默认为不显示
        defaultNan: 缺测显示值，默认缺测存成9999
        dataFormat: 返回数据格式，默认为json
        url: cmadaas的url前面固定部分，默认'http://10.40.17.54/music-ws/api?'
        SN_id: cmadaas服务节点代码，默认'NMIC_MUSIC_CMADAAS'
        ###
        """
        import os
        ## 根据参数字典，生成url
        LogLib.info('CMADaasAccess OBS output Starts: {0}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M")))
        time = CMADaasAccess.get_datetime_from_str(time)
        # print(time)
        urlstr = CMADaasAccess.combine_url_from_para(qparams, time, **kargs)
        if show: print(urlstr)
        ## 得到url返回结果
        json = CMADaasAccess.read_data_from_cmadaas(urlstr)
        ## 生成meteva数据
        sta = CMADaasAccess.dataframe_from_dict_json(json, default=defaultNan)
        ## 输出micaps3格式
        if save_path is not None:
            try:
                if filename is None: filename = "YYYYMMDDHHFF.000.m3"
                output_name = os.path.join(save_path, filename)
                output_name = meteva_base.get_path(output_name, time)
                print('OUTPUT: ', output_name)
                meteva_base.write_stadata_to_micaps3(sta, save_path=output_name, effectiveNum=2)
            except Exception as data:
                LogLib.error(
                    '{0} CMADaasAccess OBS output ERROR!: {1}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M"),
                                                                      data))
        return (sta)

    @staticmethod
    def get_files_from_cmadaas(qparams, time='202011110200', save_path=None, filename=None, show_files=True,
                               is_new=True,
                               **kargs):
        """
        time： 可以为字符串str，如“202011110200”；也可以为datetime类时间数据
        save_path: 存储文件目录，若不填写则不存储
        filename:  存储文件名,可以使用通配符(如YYYYMMDDHHFF.TTT即表示“年(四位)月(两位)日(两位)时(两位)分(两位).时效(三位)”；缺省-使用云平台原始文件名)
        show_url: 是否完整显示url，默认为不显示
        show_files: 是否完整显示待下载文件，默认为不显示
        is_new    ；True-只处理下载最新入库时间的一个文件;Fales-处理所有文件
        """
        import os
        import pandas as pd
        import datetime
        ## 根据参数字典，生成url
        time = CMADaasAccess.get_datetime_from_str(time)
        urlstr = CMADaasAccess.combine_url_from_para(qparams, time, **kargs)
        ## 得到url返回结果
        json = CMADaasAccess.read_data_from_cmadaas(urlstr)
        df = pd.DataFrame(json)
        try:
            df.sort_values(by='IYMDHM', inplace=True, ascending=False)  # 按入库时间降序排列，新更新的在上方
        except:
            print("no key IYMDHM")
        df.reset_index(drop=True, inplace=True)
        if is_new: df = df.loc[0].to_frame().T
        if show_files: print(df, type(df))

        try:
            LogLib.info('FixWriteData write_data_to_file start %s' % (save_path))
            if save_path is None: return (json)  # save_path为None，不存储文件，直接返回json信息

            for index in range(len(df)):
                fname = df.loc[index, 'FILE_NAME']
                url = df.loc[index, 'FILE_URL']
                datetime = df.loc[index, 'Datetime']
                ## url内容读取并文件保存
                file_time = CMADaasAccess.get_datetime_from_str(datetime)
                # 原url文件名
                if filename is None:
                    output_name = os.path.join(save_path, fname)
                # 自定义文件名格式
                else:
                    output_name = os.path.join(save_path, filename)
                output_name = meteva_base.get_path(output_name, file_time)
                # 文件存储
                grd, header = CMADaasAccess.read_file_from_cmadaas(url)
                print('WriteFile : ', output_name)
                CMADaasAccess.write_data_to_file(output_name, grd, creat_dir=True)

        except Exception as data:
            print("ERROR:", data)
            LogLib.error(
                '{0} CMADaasAccess Files output ERROR!: {1}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M"),
                                                                    data))

        return (json)


#


if __name__ == '__main__':
    dati_list = [datetime.datetime(2021, 12, 1, 0, 0), datetime.datetime(2021, 12, 1, 1, 0)]
    qparams = {'serviceNodeId': 'NMIC_MUSIC_CMADAAS',
               'userId': 'NMC_SGFV',
               'interfaceId': 'getSateFileByTimeRange',
               'dataCode': 'SATE_FY4A_STA_L1',
               'timeRange': CMADaasAccess.get_timerange(dati_list),
               'pwd': 'NMC_yfsjyk_601'
               }
    save_dir = r"H:\test_data\output\meb\get_files"
    urlstr = CMADaasAccess.get_files_from_cmadaas(qparams, show_url=True, save_path=save_dir)
    print(urlstr)

