from autobr.baidu import BaiduNewsSearch
from carbon2.api.submit_list_table import *

def fetch_news(config_path):
    f_in=open(config_path,"r",encoding='utf-8')
    lines=f_in.readlines()
    config={}
    for line in lines:
        line=line.strip()
        index=line.index(":")
        key=line[:index].strip()
        value=line[index+1:].strip()
        config[key]=value
        print(f"{key}\t{value}")

    save_path=config["save_path"]
    baidu_news = BaiduNewsSearch(
        webdriver_path=config["browser_path"]
    )
    list_keywords=config["keywords"].split(";")
    for k in list_keywords:
        try:
            # fetch news
            new_save_path=save_path.replace("{keyword}",k)
            baidu_news.fetch(raw_keywords=k,
                             max_pages=int(config["max_page"]),
                             silent=False,
                             save_path=new_save_path)
            # upload data
            server_url = config["server_url"]
            user_id = config["uploader"]
            csv_file = new_save_path
            save_folder =config["html_data_path"]
            submit_page_list_with_table(server_url, "dc", user_id, csv_file, save_html_folder=save_folder,
                                        use_md5url_as_id=True,
                                        driver_path=config["browser_path"], tag=config["tag"], language="zh",
                                        url_field_name='real_url', try_raise_error=True)
        except Exception as err:
            print(err)
