DEFAULT_CRAWL_CONFIG = {
    # Required
    'seed_urls' : [
    
    ],
    
    'collections_path' : '',

    'collection_name' : '',

    # Optional
    
    'crawl_filter_regexps' : [
                        
    ],

    'crawl_required_regexps' : [
    ],        
            
    'index_filter_regexps' : [
        
    ],
    
    'index_required_regexps' : [
                          
    ], 

    'max_redirects_per_url' : 10, 
    
    'max_tries_per_url' : 1, 
    
    'max_workers' : 5, 
    
    'max_saved_responses' : 20, 

    'login_url' : '',
    'login_data' : {},
    
    # Unimplemented 
    'max_hops' : 10
}

PRODUCTION_CRAWL_CONFIG = DEFAULT_CRAWL_CONFIG.copy()

PRODUCTION_CRAWL_CONFIG.update({
    'max_saved_responses' : 1000000000, 
    'max_hops' : 100,
    'max_workers' : 100
})
