import ast,os
import tokenize



abs_path=os.path.abspath(os.path.dirname(__file__))
abs_path=abs_path[:abs_path.index("extract_transform_complicate_code_new")]
import sys
sys.path.append(abs_path)
sys.path.append(abs_path+"extract_transform_complicate_code_new/")
sys.path.append(abs_path+"extract_transform_complicate_code_new/comprehension/")
import complicated_code_util
import util
import extract_compli_for_comprehension_logic
from RefactoringIdioms.extract_simp_cmpl_data import ast_util
from RefactoringIdioms.transform_c_s import transform_comprehension_dict_compli_to_simple
from tokenize import tokenize
from io import BytesIO
from pathos.multiprocessing import ProcessingPool as newPool
import traceback
'''
get comprehension node
Block=> Assign;...; For
For=>”for” ( For| If|If-else|body)
If=>”if” (body| For| If| If-else)
If-else=>”if” body “else” (body| If-else)
body=> (Assign|Call)*;Append*
并且里面只有一个写变量
for

    b.append()
    
    a.append(b)

'''

const_empty_list=[" = dict()"," = {}"]


def get_var(one_body):
    var_name=None
    if isinstance(one_body, ast.Assign):
        targets = one_body.targets
        if len(targets) > 1:
            return False

        left_node = targets[0]
        if isinstance(left_node, ast.Subscript):
            var_name=ast.unparse(left_node.value)
    return var_name



def whether_fun_is_append(one_body,assign_block_list):
    #print("one_body: ",ast.unparse(one_body),one_body.__dict__)
    if isinstance(one_body, ast.Assign):
        #print("come here")
        targets=one_body.targets
        if len(targets)>1:
            return False
        left_node=targets[0]
        if isinstance(left_node, ast.Subscript):
            assign_block_list.append([one_body])
            return True
    return False
def is_use_var(next_child, vars):
    s = ast.unparse(next_child)
    if s != list(vars)[0]:
        g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string
        for toknum, to_child, _, _, _ in g:
            if to_child.strip() == list(vars)[0]:
                return 1
    return 0
def whether_first_var_is_empty_assign(tree,for_node,vars,const_empty_list=["[]"]):
    assign_stmt=None
    assign_stmt_lineno=None
    flag=0
    # start_lineno=0
    # print("the var: ",vars)


    def whether_contain_var(node,vars):
        count=0
        s = ast.unparse(node)
        if s != list(vars)[0]:
            g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string

            for toknum, child, _, _, _ in g:
                if child == list(vars)[0]:
                    count+=1
                    # print("yes use more than one time: ", s)
        return count

    time=whether_contain_var(for_node, vars)

    append_time = 0
    for child in ast.walk(for_node):

        # if ast.unparse(child)==list(vars)[0]:
        #     time+=1
        #     if time-append_time > 0:
        #         print("come here the for node use var more than one time ",list(vars)[0])
        #         flag = 1
        #         break
        if isinstance(child,ast.Assign):
            a = []
            append_time += whether_fun_is_append(child, a)
        if isinstance(child, ast.Call):

            def is_call_use_vars(tree, fun_name, vars):
                # print(">>>>>>.come is_call_use_vars: ", fun_name)
                ana_py_fun = ast_util.Fun_Analyzer()
                ana_py_fun.visit(tree)
                for tree_fun, class_name in ana_py_fun.func_def_list:
                    if hasattr(tree_fun, "name"):
                        each_fun_name = tree_fun.name
                        # print(">>>>>>each_fun_name: ", fun_name)
                        if each_fun_name == fun_name:
                            for child in ast.walk(tree_fun):
                                if ast.unparse(child) == list(vars)[0]:
                                    # print("yes the call use vars: ", each_fun_name)
                                    return 1
                return 0

            fun_name = ast.unparse(child.func).split(".")[-1]
            call_use_var_flag = is_call_use_vars(tree, fun_name, vars)
            if call_use_var_flag:
                flag = 1
                return flag, assign_stmt_lineno, assign_stmt
        # time += whether_contain_var(child, vars)
    if time - append_time > 0:
            # print("come here the for node use var more than one time ", list(vars)[0])
            flag = 1
            return flag, assign_stmt_lineno, assign_stmt
            # break
    remove_ass_flag=1
    for node in ast.walk(tree):
        if not hasattr(node,"lineno"):
            continue

        if node.lineno==for_node.lineno:
            break
        if hasattr(node,'body') and isinstance(node.body, list) and node.lineno<for_node.lineno:
            # print("come here node: ",node,ast.unparse(node),isinstance(node,ast.Expr), )
            for child in node.body:
                if isinstance(child, ast.Assign):
                    if ast.unparse(child).strip().split("=")[0].strip() == list(vars)[0] and ast.unparse(child.value) in const_empty_list:
                        assign_stmt = child
                        assign_stmt_lineno = child.lineno
                        flag = 2


                elif assign_stmt_lineno:
                    '''
                    如果ass和for_node在同一层则不需要去除，
                    如果ass的层数在更外层，则我们需要看这一层后面是否使用了vars 如果使用了vars则ass的赋值为空的语句不可以删除
                    如果ass在更外层，在ass所在层的后面节点中，如果存在for节点，并且for节点包含该for_node，则如果for_node后使用了vars，该for_node仍然不可以简化。
                    '''
                    if isinstance(child,ast.For) and child.lineno==for_node.lineno:
                        # print("come here it does not need to remove the ass: \n",ast.unparse(tree))
                        flag=2
                        break
                        #return 2, assign_stmt_lineno, assign_stmt, remove_ass_flag

                    for next_child in ast.walk(child):
                        if not hasattr(next_child, "lineno"):
                            continue
                        if isinstance(next_child,(ast.For,ast.While,ast.AsyncFor)) and next_child.lineno<for_node.lineno and next_child.end_lineno>for_node.end_lineno:
                            # print(">>>>>>>>come here!!!!! ")
                            return 1, assign_stmt_lineno, assign_stmt, remove_ass_flag

                            # for for_child in ast.walk(next_child):
                                # if hasattr(for_child,'lineno'):
                                #     if for_child.lineno>for_node.end_lineno:
                                #         if is_use_var(for_child,vars):
                                #             flag=1
                                #             print(">>>>>>>>come here ")
                                #             return flag, assign_stmt_lineno, assign_stmt, remove_ass_flag
                        elif  next_child.lineno>for_node.end_lineno:
                            if is_use_var(next_child, vars):
                                remove_ass_flag = 0
                                # break

        if assign_stmt_lineno and node!=assign_stmt and node.end_lineno<for_node.lineno and node.lineno>assign_stmt_lineno :
            # print("come here")

            if is_use_var(node, vars):
                flag = 1
                break
            # if s!=list(vars)[0]:
            #     g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string
            #     for toknum, child, _, _, _ in g:
            #         if child.strip() == list(vars)[0]:
            #             flag = 1
            #             # print("yes use more than one time: ",vars,s,"\n>>>>>>>>>",ast.unparse(for_node))
            #             break
            #     if flag==1:
            #         break

    return flag,assign_stmt_lineno,assign_stmt,remove_ass_flag
'''
def whether_first_var_is_empty_assign(tree,for_node,vars,const_empty_list=["[]"]):
    assign_stmt=None
    assign_stmt_lineno=None
    flag=0
    # start_lineno=0
    # print("the var: ",vars)


    def whether_contain_var(node,vars):
        count=0
        s = ast.unparse(node)
        if s != list(vars)[0]:
            g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string

            for toknum, child, _, _, _ in g:
                if child == list(vars)[0]:
                    count+=1
                    # print("yes use more than one time: ", s)
        return count

    time=whether_contain_var(for_node, vars)
    append_time = 0
    for child in ast.walk(for_node):

        # if ast.unparse(child)==list(vars)[0]:
        #     time+=1
        #     if time-append_time > 0:
        #         print("come here the for node use var more than one time ",list(vars)[0])
        #         flag = 1
        #         break
        if isinstance(child,ast.Assign):
            a = []
            append_time += whether_fun_is_append(child, a)
        if isinstance(child, ast.Call):



            def is_call_use_vars(tree, fun_name, vars):
                # print(">>>>>>.come is_call_use_vars: ", fun_name)
                ana_py_fun = ast_util.Fun_Analyzer()
                ana_py_fun.visit(tree)
                for tree_fun, class_name in ana_py_fun.func_def_list:
                    if hasattr(tree_fun,"name"):
                        each_fun_name = tree_fun.name
                        # print(">>>>>>each_fun_name: ", fun_name)
                        if each_fun_name == fun_name:
                            for child in ast.walk(tree_fun):
                                if ast.unparse(child) == list(vars)[0]:
                                    # print("yes the call use vars: ", each_fun_name)
                                    return 1
                return 0

            fun_name = ast.unparse(child.func).split(".")[-1]
            call_use_var_flag = is_call_use_vars(tree, fun_name, vars)
            if call_use_var_flag:
                flag = 1
                return flag, assign_stmt_lineno, assign_stmt
        # time += whether_contain_var(child, vars)
    if time - append_time > 0:
            # print("come here the for node use var more than one time ", list(vars)[0])
            flag = 1
            return flag, assign_stmt_lineno, assign_stmt
            # break
    for node in ast.walk(tree):
        if not hasattr(node,"lineno"):
            continue

        if node.lineno==for_node.lineno:
            break
        if isinstance(node, ast.Assign):
            # print("the assign: ",ast.unparse(node),ast.unparse(node).strip().split("=")[0].strip())
            if ast.unparse(node).strip().split("=")[0].strip()==list(vars)[0] and ast.unparse(node.value) in const_empty_list:
                assign_stmt=node
                assign_stmt_lineno=node.lineno
                flag = 2
        if assign_stmt_lineno and node.end_lineno<for_node.lineno and node.lineno>assign_stmt_lineno :
            s=ast.unparse(node)

            if s!=list(vars)[0]:
                g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string
                for toknum, child, _, _, _ in g:
                    if child.strip() == list(vars)[0]:
                        flag = 1
                        # print("yes use more than one time: ",vars,s,"\n>>>>>>>>>",ast.unparse(for_node))
                        break
                if flag==1:
                    break
    return flag,assign_stmt_lineno,assign_stmt
'''
def else_traverse(if_body,assign_block_list):
    else_body_list = if_body.orelse
    if len(else_body_list) > 1:
        return False
    else:
        else_body_one = else_body_list[0]
        if isinstance(else_body_one, ast.If):
            or_else = else_body_one.orelse
            if not or_else:
                return False
            else:
                return if_else_traverse(else_body_one,assign_block_list)
        return whether_fun_is_append(else_body_one,assign_block_list)



def if_else_traverse(if_body,assign_block_list):
    if_body_list=if_body.body
    # else_body_list=if_body.orelse
    if len(if_body_list)==1:
        if_body_flag =whether_fun_is_append(if_body_list[0],assign_block_list)
    else:
        return False
    #print("if body flag: ",if_body_flag)

    return if_body_flag and else_traverse(if_body,assign_block_list)

def if_traverse(one_body,assign_block_list):
    orelse = one_body.orelse
    if_body=one_body.body
    if not orelse:
        return for_traverse(one_body,assign_block_list)
    else:
        #print("it is if-else node: ", ast.unparse(one_body))
        return if_else_traverse(one_body,assign_block_list)


def for_traverse(node,assign_block_list):
    for_body_list = node.body
    if isinstance(node, ast.For) and node.orelse:
        #print("come here")
        return False
    if len(for_body_list) == 1:
        one_body=for_body_list[0]
        if isinstance(one_body, ast.For):
            return for_traverse(one_body,assign_block_list)
        elif isinstance(one_body, ast.If):
            #print("it is if node")
            return if_traverse(one_body,assign_block_list)

        elif isinstance(one_body, ast.Assign):
            #print("it is one stmt")
            return whether_fun_is_append(one_body,assign_block_list)
        else:

            #print("the for/if sentences has other stmts so cannot transform such as try stmt", ast.unparse(one_body),type(one_body))
            return False
    else:
        return False
'''
beg1 end1
beg2 end2
'''


def get_complicated_for_comprehen_code_list(tree,content):

    #tree = ast.parse(code1)
    code_index_start_end_list = []

    start_lineno=1
    #print("tree: ",tree.body[0].__dict__)
    start_lineno=tree.body[0].lineno
    #print("start_lineno: ",start_lineno)
    #code1 = "\n".join(content.split("\n")[start_lineno-1:])

    for ind_line,node in enumerate(ast.walk(tree)):

        if isinstance(node, ast.For):
            #print("node_line: ",ast.unparse(node),node.lineno, node.end_lineno)
            # if node.lineno!=1674:
            #     continue

            assign_block_list=[]
            #print("it is what: ",node.lineno)
            if for_traverse(node,assign_block_list):
                #print("maybe comrephension")
                vars = set([])
                #print("maybe comrephension: ",node.lineno,ast.unparse(assign_block_list[0][0]))
                for each_block in assign_block_list:
                    var_name=get_var(each_block[0])
                    vars.add(var_name)
                if len(vars)!=1:
                    continue
                # print("vars: ",vars,ind_line,start_lineno)
                # print("node: ", ast.unparse(node))
                flag,assign_stmt_lineno,assign_stmt,remove_ass_flag=whether_first_var_is_empty_assign(tree, node, vars,["dict()","{}"])
                # print("flag: ", flag)
                if flag!=2:
                    print("flag: ")#, ast.unparse(node)
                    continue

                assign_block_list_str = []
                for one_block in assign_block_list:
                    one_block_str = []
                    for e in one_block:
                        one_block_str.append(ast.unparse(e))
                    assign_block_list_str.append(one_block_str)
                code_index_start_end_list.append(
                        [node, assign_stmt,remove_ass_flag])#, node.lineno, node.end_lineno, assign_stmt_lineno, assign_block_list_str])

                # print(">>>>>>>>>>>>>>>>>>>>>>it is comprehension code1\n", remove_ass_flag,node.lineno, node.end_lineno,assign_stmt_lineno,ast.unparse(node))
                # code_index_start_end_list.append([ast.unparse(node), node.lineno, node.end_lineno,assign_stmt_lineno,assign_block_list_str])
    #print("len: ", len(code_index_start_end_list))
    code_index_start_end_list = extract_compli_for_comprehension_logic.filter_overlap(code_index_start_end_list)
    # code_index_start_end_list = filter_overlap(code_index_start_end_list)
    # print("len: ",len(code_index_start_end_list))
    return code_index_start_end_list
def save_one_repo(repo_name):
    count=0
    one_repo_chained_comparison_code_list = []
    dict_file = dict()
    for file_info in dict_repo_file_python[repo_name]:
        file_path = file_info["file_path"]
        file_html = file_info["file_html"]
        # if file_html != "https://github.com/networkx/networkx/tree/master/networkx/algorithms/connectivity/kcomponents.py":#"https://github.com/IDSIA/sacred/tree/master/sacred/observers/mongo.py":  # "https://github.com/cloud-custodian/cloud-custodian/tree/master/c7n/provider.py":#"https://github.com/networkx/networkx/tree/master/networkx/generators/joint_degree_seq.py":#"https://github.com/bndr/pipreqs/tree/master/pipreqs/pipreqs.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/networkx/networkx/tree/master/networkx/readwrite/json_graph/adjacency.py":##"https://github.com/microsoft/nni/tree/master/nni/tools/nnictl/nnictl_utils.py":#"https://github.com/networkx/networkx/tree/master/networkx/readwrite/json_graph/adjacency.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/aws/aws-cli/tree/master/awscli/customizations/s3/subcommands.py":#"https://github.com/nccgroup/ScoutSuite/tree/master/ScoutSuite/providers/aws/utils.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":#"https://github.com/localstack/localstack/tree/master/localstack/services/awslambda/lambda_api.py":#"https://github.com/localstack/localstack/tree/master/localstack/utils/common.py":#"https://github.com/aws/aws-cli/tree/master/awscli/compat.py":#"https://github.com/spulec/moto/tree/master/moto/config/models.py":#"https://github.com/cookiecutter/cookiecutter/tree/master/cookiecutter/hooks.py":#"https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/utils.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/OmkarPathak/pygorithm/tree/master/pygorithm/greedy_algorithm/fractional_knapsack.py":#"https://github.com/OmkarPathak/pygorithm/tree/master/pygorithm/strings/anagram.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/cloudtools/troposphere/tree/master/troposphere/validators.py":#"https://github.com/pytransitions/transitions/tree/master/transitions/extensions/nesting.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/sympy/sympy/tree/master/sympy/crypto/crypto.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/sympy/sympy/tree/master/sympy/crypto/crypto.py":#"https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/utils.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/pytransitions/transitions/tree/master/transitions/extensions/nesting.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":  # "https://github.com/pymc-devs/pymc/tree/master/pymc/sampling.py":#"https://github.com/smicallef/spiderfoot/tree/master//sflib.py":#"https://github.com/smicallef/spiderfoot/tree/master//sfwebui.py":#
        #     continue
        # if file_html != "https://github.com/networkx/networkx/tree/master/networkx/generators/joint_degree_seq.py":  #"https://github.com/cloud-custodian/cloud-custodian/tree/master/c7n/provider.py":  # "https://github.com/networkx/networkx/tree/master/networkx/generators/joint_degree_seq.py":#"https://github.com/bndr/pipreqs/tree/master/pipreqs/pipreqs.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/networkx/networkx/tree/master/networkx/readwrite/json_graph/adjacency.py":##"https://github.com/microsoft/nni/tree/master/nni/tools/nnictl/nnictl_utils.py":#"https://github.com/networkx/networkx/tree/master/networkx/readwrite/json_graph/adjacency.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/aws/aws-cli/tree/master/awscli/customizations/s3/subcommands.py":#"https://github.com/nccgroup/ScoutSuite/tree/master/ScoutSuite/providers/aws/utils.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":#"https://github.com/localstack/localstack/tree/master/localstack/services/awslambda/lambda_api.py":#"https://github.com/localstack/localstack/tree/master/localstack/utils/common.py":#"https://github.com/aws/aws-cli/tree/master/awscli/compat.py":#"https://github.com/spulec/moto/tree/master/moto/config/models.py":#"https://github.com/cookiecutter/cookiecutter/tree/master/cookiecutter/hooks.py":#"https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/utils.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/OmkarPathak/pygorithm/tree/master/pygorithm/greedy_algorithm/fractional_knapsack.py":#"https://github.com/OmkarPathak/pygorithm/tree/master/pygorithm/strings/anagram.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/cloudtools/troposphere/tree/master/troposphere/validators.py":#"https://github.com/pytransitions/transitions/tree/master/transitions/extensions/nesting.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/sympy/sympy/tree/master/sympy/crypto/crypto.py":#"https://github.com/ytdl-org/youtube-dl/tree/master/youtube_dl/utils.py":#"https://github.com/sympy/sympy/tree/master/sympy/crypto/crypto.py":#"https://github.com/yt-dlp/yt-dlp/tree/master/yt_dlp/utils.py":#"https://github.com/amperser/proselint/tree/master/proselint/tools.py":#"https://github.com/pytransitions/transitions/tree/master/transitions/extensions/nesting.py":#"https://github.com/microsoft/nni/tree/master/nni/algorithms/hpo/networkmorphism_tuner/graph_transformer.py":  # "https://github.com/pymc-devs/pymc/tree/master/pymc/sampling.py":#"https://github.com/smicallef/spiderfoot/tree/master//sflib.py":#"https://github.com/smicallef/spiderfoot/tree/master//sfwebui.py":#
        #     continue
        # if file_html!="https://github.com/python-poetry/poetry/tree/master/poetry/puzzle/solver.py":#"https://github.com/Rockyzsu/stock/tree/master//fupan.py":#"https://github.com/OctoPrint/OctoPrint/tree/master/src/octoprint/server/views.py":
        #     continue
        print("file_html: ", file_html)
        try:
            content = util.load_file_path(file_path)
        except:
            print(f"{file_path} is not existed!")
            continue

        try:
            file_tree = ast.parse(content)
            ana_py = ast_util.Fun_Analyzer()
            ana_py.visit(file_tree)
            # one_file_chained_comparison_code_list = []
            #print("func number: ",file_html,len(ana_py.func_def_list))
            dict_class = dict()
            for tree, class_name in ana_py.func_def_list:
                # if 1:
                #print("come here",ast.unparse(tree))
                new_code_list = get_complicated_for_comprehen_code_list(tree,content)
                for ind, (for_node, assign_node,remove_ass_flag) in enumerate(new_code_list):
                    new_code_list[ind].append(
                        transform_comprehension_dict_compli_to_simple.transform(for_node, assign_node))
                # for old_code,old_ass,new_code in new_code_list:
                #     print("old_code,old_ass,new_code: ",ast.unparse(old_code),old_ass,ast.unparse(new_code))

                # one_file_chained_comparison_code_list.extend(get_complicated_for_comprehen_code_list(tree,content))
                ast_util.set_dict_class_code_list(tree, dict_class, class_name, new_code_list)

                # count+=len(one_file_chained_comparison_code_list)
            dict_file[file_html] = dict_class
            #print(dict_class)
            # if one_file_chained_comparison_code_list:
            #     one_repo_chained_comparison_code_list.append(
            #         [one_file_chained_comparison_code_list, file_path, file_html])
                # print("one_file_truth_value_test_code_list: ",one_repo_chained_comparison_code_list)
                    # break
        except SyntaxError:
            print("the file has syntax error")
            continue
        except ValueError:
            print("the file has value error: ", file_html)#content,
            continue
        except:
            traceback.print_exc()
        # break
    #print("count: ",count)
    util.save_pkl(save_complicated_code_dir_pkl, repo_name, dict_file)
    print("save successfully! ", count,save_complicated_code_dir_pkl + repo_name)


if __name__ == '__main__':
    code = '''
# def func_1():
#     components = {}
#     for i in range(2):
#         print(1)
#     components['trend'] = get_forecast_component_plotly_props(
#         m, fcst, 'trend', uncertainty, plot_cap)
#     if m.train_holiday_names is not None and 'holidays' in fcst:
#         components['holidays'] = get_forecast_component_plotly_props(
#             m, fcst, 'holidays', uncertainty)
#     
#     regressors = {'additive': False, 'multiplicative': False}
#     for name, props in m.extra_regressors.items():
#         regressors[props['mode']] = True
#     for mode in ['additive', 'multiplicative']:
#         if regressors[mode] and 'extra_regressors_{}'.format(mode) in fcst:
#             components['extra_regressors_{}'.format(mode)] = get_forecast_component_plotly_props(
#                 m, fcst, 'extra_regressors_{}'.format(mode))
payload={}
for field in model.__searchable__:
    payload[field] = getattr(model, field)
'''
    # dict_repo_file_python = util.load_json(util.data_root, "python3_repos_files_info")
    #
    # save_one_repo("Real-Time-Voice-Cloning", code1)

    '''
    # print("code1: ",code1)
    tree = ast.parse(code1)
    #code1 = ast.unparse(tree)
    #print("code1: ", code1)

    code_list=get_complicated_for_comprehen_code_list(tree,code1)
    print("len: ",len(code_list))
    for e in code_list:
        print(">>>>>code1： ", e[0])
        print("block: ", e[-1])
        for each in  e[-1]:
            print(each)

        print("----------END------------")
    '''
    # test_code="a=dict()"
    # test_tree = ast.parse(test_code)
    # print(ast.unparse(test_tree))

    # layer_node_list = []
    # ast_util.extract_ast_cur_layer_node(tree, layer_node_list)
    '''
    get comprehension node
    Block=> Assign;...; For
    For=>”for” ( For| If|If-else|body)
    If=>”if” (body| For| If| If-else)
    If-else=>”if” body “else” (body| If-else)
    body=> (Assign|Call)*;Append*
    '''
    # 对于
    save_complicated_code_dir_pkl = util.data_root + "transform_complicate_to_simple_pkl/for_compre_dict/"
    # save_complicated_code_dir = util.data_root + "complicated_code_dir/for_comrephension_dict_complicated_only_one_stmt/"
    dict_repo_file_python = util.load_json(util.data_root, "python3_1000repos_files_info")

    #'''
    # dict_repo_file_python = util.load_json(util.data_root, "python3_1000repos_files_info")
    repo_list = []
    for ind, repo_name in enumerate(dict_repo_file_python):
        # if repo_name!="networkx":#"sacred":#"networkx":#"cloud-custodian":#"poetry":#"OctoPrint":#"keras-bert":#"Legofy":
        #     continue
        # print("repo_name: ", repo_name)
        repo_list.append(repo_name)
    # save_one_repo(repo_list[0])

    #'''
    pool = newPool(nodes=30)
    pool.map(save_one_repo, repo_list)  # [:3]sample_repo_url ,token_num_list[:1]
    pool.close()
    pool.join()
    #'''

    files_num_list = []
    star_num_list = []
    contributor_num_list = []
    count_repo, file_count, me_count, code_count = 0, 0, 0, 0
    file_list = set([])
    repo_code_num = dict()
    result_compli_for_else_list = []
    all_count_repo, all_file_count, all_me_count = 0, 0, 0
    for file_name in os.listdir(save_complicated_code_dir_pkl):
        all_count_repo += 1
        repo_name = file_name[:-4]
        # files_num_list.append(repo_files_info[repo_name])
        # star_num_list.append(repo_star_info[repo_name])
        # contributor_num_list.append(repo_contributor_info[repo_name])

        complicate_code = util.load_pkl(save_complicated_code_dir_pkl, repo_name)

        repo_file_count, repo_me_count, repo_code_count, repo_all_file_count, repo_all_me_count = complicated_code_util.get_code_count(
            complicate_code)
        # for code_list, file_path, file_html in complicate_code:
        code_count += repo_code_count
        file_count += repo_file_count
        me_count += repo_me_count
        all_file_count += repo_all_file_count
        all_me_count += repo_all_me_count
        repo_exist = 0
        for file_html in complicate_code:
            for cl in complicate_code[file_html]:
                for me in complicate_code[file_html][cl]:
                    if complicate_code[file_html][cl][me]:
                        repo_exist = 1
                        for code in complicate_code[file_html][cl][me]:
                            # print("html: ",file_html,cl,me,ast.unparse(code1[0]))
                            # print("code1",code1)
                            result_compli_for_else_list.append(
                                [repo_name, file_html, cl, me, ast.unparse(code[0]), ast.unparse(code[1])])

            # print(f"{file_html} of {repo_name} has  {len(code_list)} code1 fragments")
        count_repo += repo_exist

    # a=dict(sorted(repo_code_num.items(), key=lambda item: item[1], reverse=True))
    # print(a)
    # print(np.median(list(a.values())), np.max(list(a.values())), np.min(list(a.values())))
    # print(np.median(files_num_list), np.max(files_num_list), np.min(files_num_list))
    # print(np.median(star_num_list), np.max(star_num_list), np.min(star_num_list))
    # print(np.median(contributor_num_list), np.max(contributor_num_list), np.min(contributor_num_list))
    print("count: ", count_repo, code_count, file_count, me_count, all_count_repo, all_file_count, all_me_count)
    # 1 156 2943 100 1 2990 40102 salt
    # 791 2010 103165 1291 800 121348 1192868
    # util.save_csv(util.data_root + "complicated_code_dir_pkl/for_compre_dict.csv", result_compli_for_else_list,
    #               ["repo_name", "file_html", "class_name", "me_name", "for_code", "assign_code"])











