import sys

sys.path.insert(0, '../../../')
from gxl_ai_utils.utils import utils_file

# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata/yhdai/multi_task_stage021/mix_task/Speech_Age_Sex/age_sex/shards_list.txt"
# dataname = "Speech_Age_Sex"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage021/mix_task/Speech_Age_Sex/age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=6 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Speech_Style_Age_Sex/speech_style_age_sex/shards_list.txt"
# dataname = "Speech_Style_Age_Sex"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Speech_Style_Age_Sex/speech_style_age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=6 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Age_Sex/age_sex/shards_list.txt"
# dataname = "Age_Sex"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Age_Sex/age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=6 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Style_Age_Sex/style_age_sex/shards_list.txt"
# dataname = "Style_Age_Sex"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Style_Age_Sex/style_age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=6 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")

# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node60_tmpdata/3500_chat/shards_list.txt"
# dataname = "3500_chat"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Style_Age_Sex/style_age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=12
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


#
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node60_tmpdata/3500_asr/shards_list.txt"
# dataname = "3500_asr"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node54_tmpdata/yhdai/multi_task_stage022/mix_task/Style_Age_Sex/style_age_sex/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=12
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs9/wjtian/0107_esc50-allclass_test.jsonl"
# dataname = "caption_0107_esc50"
# dict_list = utils_file.load_dict_list_from_jsonl(input_shards_path)
# output_wav_list_path = utils_file.do_get_fake_file()
# wav_list = []
# for dict_i in dict_list:
#     wav_list.append(dict_i['wav'])
# utils_file.write_list_to_file(wav_list, output_wav_list_path)
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=output_wav_list_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/wav",
#     num_thread=4
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs9/wjtian/0107_vocalsound_test.jsonl"
# dataname = "caption_0107_vocalsound"
# dict_list = utils_file.load_dict_list_from_jsonl(input_shards_path)
# output_wav_list_path = utils_file.do_get_fake_file()
# wav_list = []
# for dict_i in dict_list:
#     wav_list.append(dict_i['wav'])
# utils_file.write_list_to_file(wav_list, output_wav_list_path)
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=output_wav_list_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/wav",
#     num_thread=4
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")

# time_now = utils_file.do_get_now_time()
# dataname = "caption_0107_esc50"
# input_shards_path = f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/data.list"
# old_path = f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/data_old.list"
# dict_list = utils_file.load_dict_list_from_jsonl(input_shards_path)
# new_dict_list = []
# for dict_i in dict_list:
#     dict_i['wav'] = f'/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/wav/{dict_i["key"]}.wav'
#     new_dict_list.append(dict_i)
# utils_file.copy_file(input_shards_path, old_path)
# utils_file.write_dict_list_to_jsonl(new_dict_list, input_shards_path)
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
#
# time_now = utils_file.do_get_now_time()
# dataname = "caption_0107_vocalsound"
# input_shards_path = f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/data.list"
# old_path = f"/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/data_old.list"
# dict_list = utils_file.load_dict_list_from_jsonl(input_shards_path)
# new_dict_list = []
# for dict_i in dict_list:
#     dict_i['wav'] = f'/mnt/sfs/asr/test_data/test_sets_format_3000/{dataname}/wav/{dict_i["key"]}.wav'
#     new_dict_list.append(dict_i)
# utils_file.copy_file(input_shards_path, old_path)
# utils_file.write_dict_list_to_jsonl(new_dict_list, input_shards_path)
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs9/yacao/nfs7_copy/yacao/shards/0114_wjtian_simu2/nospeech/shards_list.txt"
# dataname = "caption_nospeech_add_2025_1_15"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/work_nfs9/yacao/nfs7_copy/yacao/shards/0114_wjtian_simu2/nospeech/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=6
# )
# esl_time = utils_file.do_get_elapsed_time(time_[now)
# utils_file.logging_info(f"Total time: {esl_time} s")
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/38_data/znlin/ASR_align/data.list"
# root_huawei_dir = "/mnt/sfs/asr/test_data/test_sets_format_3000"
# dataname = "aligin_english_clean"
# item_root_dir = f"{root_huawei_dir}/{dataname}"
# # 得到huawei的data.list text
# dict_list = utils_file.load_dict_list_from_jsonl(input_shards_path)
# text_dict = {}
# wav_path_list = []
# for dict_i in dict_list:
#     tmp_wav_path = dict_i['wav']
#     wav_path_list.append(tmp_wav_path)
#     dict_i['wav'] = f"{item_root_dir}/{tmp_wav_path.split('/')[-1]}"
#     text_dict[dict_i['key']] = dict_i['txt']
# fake_data_list_path = utils_file.do_get_fake_file()
# utils_file.write_dict_list_to_jsonl(dict_list, fake_data_list_path)
# fake_text_path = utils_file.do_get_fake_file()
# utils_file.write_dict_to_scp(text_dict, fake_text_path)
#
#
# utils_file.do_sync_copy_file_upload(fake_data_list_path,f"{item_root_dir}/data.list",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_copy_file_upload(fake_text_path,f"{item_root_dir}/text",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# fake_wav_paths_path= utils_file.do_get_fake_file()
# utils_file.write_list_to_file(wav_path_list, fake_wav_paths_path)
#
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=fake_wav_paths_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=10
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")

# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs12/znlin/whale-kws/forTest_cn/wav.scp.new"
#
# # 得到fake data.list
# wav_dict  = utils_file.load_dict_from_scp(input_shards_path)
# dict_list_tmp = []
# for key, path in wav_dict.items():
#     dict_list_tmp.append({'key': key, 'wav': path, 'txt': 'None'})
# fake_data_list_path_local = utils_file.do_get_fake_file()
# utils_file.write_dict_list_to_jsonl(dict_list_tmp, fake_data_list_path_local)
# # sys.exit(0)
# root_huawei_dir = "/mnt/sfs/asr/test_data/test_sets_format_3000"
# dataname = "align_cn_noize"
# item_root_dir = f"{root_huawei_dir}/{dataname}"
# # 得到huawei的data.list text
# dict_list = utils_file.load_dict_list_from_jsonl(fake_data_list_path_local)
# text_dict = {}
# wav_path_list = []
# for dict_i in dict_list:
#     tmp_wav_path = dict_i['wav']
#     wav_path_list.append(tmp_wav_path)
#     dict_i['wav'] = f"{item_root_dir}/{tmp_wav_path.split('/')[-1]}"
#     text_dict[dict_i['key']] = dict_i['txt']
# fake_data_list_path = utils_file.do_get_fake_file()
# utils_file.write_dict_list_to_jsonl(dict_list, fake_data_list_path)
# fake_text_path = utils_file.do_get_fake_file()
# utils_file.write_dict_to_scp(text_dict, fake_text_path)
#
#
# utils_file.do_sync_copy_file_upload(fake_data_list_path,f"{item_root_dir}/data.list",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_copy_file_upload(fake_text_path,f"{item_root_dir}/text",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# fake_wav_paths_path= utils_file.do_get_fake_file()
# utils_file.write_list_to_file(wav_path_list, fake_wav_paths_path)
#
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=fake_wav_paths_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=10
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")




# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs12/znlin/whale-kws/forTest_en/wav.scp.new"
# text_path = "/home/38_data/znlin/ASR_align/text"
#
# # 得到fake data.list
# wav_dict  = utils_file.load_dict_from_scp(input_shards_path)
# text_dict = utils_file.load_dict_from_scp(text_path)
# dict_list_tmp = []
# for key, path in wav_dict.items():
#     dict_list_tmp.append({'key': key, 'wav': path, 'txt': text_dict.get(key, 'None')})
# fake_data_list_path_local = utils_file.do_get_fake_file()
# utils_file.write_dict_list_to_jsonl(dict_list_tmp, fake_data_list_path_local)
#
# root_huawei_dir = "/mnt/sfs/asr/test_data/test_sets_format_3000"
# dataname = "align_en_noize"
# item_root_dir = f"{root_huawei_dir}/{dataname}"
# # 得到huawei的data.list text
# dict_list = utils_file.load_dict_list_from_jsonl(fake_data_list_path_local)
# text_dict = {}
# wav_path_list = []
# for dict_i in dict_list:
#     tmp_wav_path = dict_i['wav']
#     wav_path_list.append(tmp_wav_path)
#     dict_i['wav'] = f"{item_root_dir}/{tmp_wav_path.split('/')[-1]}"
#     text_dict[dict_i['key']] = dict_i['txt']
# fake_data_list_path = utils_file.do_get_fake_file()
# utils_file.write_dict_list_to_jsonl(dict_list, fake_data_list_path)
# fake_text_path = utils_file.do_get_fake_file()
# utils_file.write_dict_to_scp(text_dict, fake_text_path)
#
#
# utils_file.do_sync_copy_file_upload(fake_data_list_path,f"{item_root_dir}/data.list",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_copy_file_upload(fake_text_path,f"{item_root_dir}/text",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# fake_wav_paths_path= utils_file.do_get_fake_file()
# utils_file.write_list_to_file(wav_path_list, fake_wav_paths_path)
#
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=fake_wav_paths_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=10
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")




# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node57_data/hkxie/4O/speech_data_final2/TEXT2TOKENv1/shards_list.txt"
# dataname = "TEXT2TOKENv1_cosivoice1_text2token_add_2025_2_16"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,'/home/node57_data/hkxie/4O/speech_data_final2/TEXT2TOKENv1/shards',f"/mnt/sfs/asr/update_data/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"/mnt/sfs/asr/update_data/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"/mnt/sfs/asr/update_data/{dataname}",
#     num_thread=8 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")

# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs16/asr_data/data/dialect/shards_list.txt"
# dataname = "dialect_new_add_2025_2_27"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs16/asr_data/data/dialect/shards/{dataname}/shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=10 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")



# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata2/gjli/style_age_gender_data/style_labeling_100wto200w_part1_age_gender/shards_list.txt"
# dataname = "style_labeling_100wto200w_part1_age_gender"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node54_tmpdata2/gjli/style_age_gender_data/style_labeling_100wto200w_part1_age_gender',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")



# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata2/gjli/style_age_gender_data/style_origin_tts_age_gender/shards_list.txt"
# dataname = "style_origin_tts_age_gender"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node54_tmpdata2/gjli/style_age_gender_data/style_origin_tts_age_gender',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata2/gjli/style_age_gender_data/style_origin_tts_age_gender/shards_list.txt"
# dataname = "style_origin_tts_age_gender"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node54_tmpdata2/gjli/style_age_gender_data/style_origin_tts_age_gender',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")



# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs16/emotion_data/OSUM_age_gender/emotion_age_gender1/shards_list.txt"
# dataname = "emotion_age_gender1"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs16/emotion_data/OSUM_age_gender/emotion_age_gender1/shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")

# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/node54_tmpdata2/znlin/ASR-speaker/DB-ASR-106/shards_list.txt"
# dataname = "DB-ASR-106_speaker_add_2025_3_5"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node54_tmpdata2/znlin/ASR-speaker/DB-ASR-106/shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
#
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs15/asr_data/data/ASR-speaker/Ali_near/shards_list.txt"
# dataname = "Ali_near_speaker_add_2025_3_5"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs15/asr_data/data/ASR-speaker/Ali_near/shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
#
#
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs15/asr_data/data/ASR-speaker/LibriMix/shards_list.txt"
# dataname = "LibriMix_speaker_add_2025_3_5"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs15/asr_data/data/ASR-speaker/Ali_near/shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs16/asr_data/data/dialect/shards_list2.txt"
# dataname = "dialect_next_new_add_2025_3_5"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs16/asr_data/data/dialect/shards2',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")
#
#
#
#
#
# # -----------
# time_now = utils_file.do_get_now_time()
# input_shards_path = "/home/work_nfs16/asr_data/data/punctuation/test_shards_list.txt"
# dataname = "punctuation_test_add_2025_3_5"
# remote_root_dir="/mnt/sfs/asr/update_data/multi_task"
# fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/work_nfs16/asr_data/data/punctuation/test_shards',f"{remote_root_dir}/{dataname}")
# utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
#                                     username="root",
#                                     password="Fy!mATB@QE",
#                                     remote_host="139.210.101.41",
#                                     )
# utils_file.do_sync_files_upload_data_multi_thread(
#     file_list_path=input_shards_path,
#     username="root",
#     password="Fy!mATB@QE",
#     remote_host="139.210.101.41",
#     remote_dir=f"{remote_root_dir}/{dataname}",
#     num_thread=15 # 30h
# )
# esl_time = utils_file.do_get_elapsed_time(time_now)
# utils_file.logging_info(f"Total time: {esl_time} s")


time_now = utils_file.do_get_now_time()
input_shards_path = "/home/node48_tmpdata/hkxie/4O/speech_data_final/Speech2TEXTandTOKEN_part_0/shards_list.txt"
dataname = "Speech2TEXTandTOKEN_part_0_from_chat_text_add_2025_3_17"
remote_root_dir="/mnt/sfs/asr/update_data"
fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node48_tmpdata/hkxie/4O/speech_data_final/Speech2TEXTandTOKEN_part_0/shards',f"{remote_root_dir}/{dataname}")
utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
                                    username="root",
                                    password="Fy!mATB@QE",
                                    remote_host="139.210.101.41",
                                    )
utils_file.do_sync_files_upload_data_multi_thread(
    file_list_path=input_shards_path,
    username="root",
    password="Fy!mATB@QE",
    remote_host="139.210.101.41",
    remote_dir=f"{remote_root_dir}/{dataname}",
    num_thread=5 # 30h
)
esl_time = utils_file.do_get_elapsed_time(time_now)
utils_file.logging_info(f"Total time: {esl_time} s")

time_now = utils_file.do_get_now_time()
input_shards_path = "/home/node48_tmpdata/hkxie/4O/speech_data_final/TEXT2TOKEN_hq/shards_list.txt"
dataname = "TEXT2TOKEN_hq_add_2025_3_17"
remote_root_dir="/mnt/sfs/asr/update_data"
fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node48_tmpdata/hkxie/4O/speech_data_final/TEXT2TOKEN_hq/shards',f"{remote_root_dir}/{dataname}")
utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
                                    username="root",
                                    password="Fy!mATB@QE",
                                    remote_host="139.210.101.41",
                                    )
utils_file.do_sync_files_upload_data_multi_thread(
    file_list_path=input_shards_path,
    username="root",
    password="Fy!mATB@QE",
    remote_host="139.210.101.41",
    remote_dir=f"{remote_root_dir}/{dataname}",
    num_thread=5 # 30h
)
esl_time = utils_file.do_get_elapsed_time(time_now)
utils_file.logging_info(f"Total time: {esl_time} s")


time_now = utils_file.do_get_now_time()
input_shards_path = "/home/node48_tmpdata/hkxie/4O/speech_data_final/TEXT2TOKEN_hq/shards_list.txt"
dataname = "TEXT2TOKEN_hq_add_2025_3_17"
remote_root_dir="/mnt/sfs/asr/update_data"
fake_file = utils_file.do_replace_str_for_file_and_return_new_file(input_shards_path,f'/home/node48_tmpdata/hkxie/4O/speech_data_final/TEXT2TOKEN_hq/shards',f"{remote_root_dir}/{dataname}")
utils_file.do_sync_copy_file_upload(fake_file,f"{remote_root_dir}/{dataname}/shards_list.txt",
                                    username="root",
                                    password="Fy!mATB@QE",
                                    remote_host="139.210.101.41",
                                    )
utils_file.do_sync_files_upload_data_multi_thread(
    file_list_path=input_shards_path,
    username="root",
    password="Fy!mATB@QE",
    remote_host="139.210.101.41",
    remote_dir=f"{remote_root_dir}/{dataname}",
    num_thread=5 # 30h
)
esl_time = utils_file.do_get_elapsed_time(time_now)
utils_file.logging_info(f"Total time: {esl_time} s")