
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

import config

# 保存文件夹名
save_dir = 'save_dir'
type_name = 'type_name'

basic_setting = config.load_config(config.BASIC_SETTING_FILE, config.BASIC_SETTING_DEFAULT_CONFIG)
basic_setting["save_dir"] = save_dir
basic_setting["type_name"] = type_name
config.save_config(config.BASIC_SETTING_FILE, basic_setting)

# 获取项目设置
settings = get_project_settings()

keywords = ['t1', 't2']

for keyword in keywords:
    # 创建 CrawlerProcess 实例
    process = CrawlerProcess(settings)

    process.crawl("spider1", keyword=keyword, pages=100)

    process.crawl("spider2", keyword=keyword, pages=40)

    # 开始运行
    process.start()
