ignore_tags = img,script,link
reject =  *.jpg,*.jpeg,*.xml,*.svg,*.tif,*.tiff,*.gif,*.iso,*.js,*.css,*.zip,*.tar.gz,*.tar,*.tgz,*.gz,*.bz,*.bz2,*.rar,*.7z,*.wav,*.mp3,*.mp4,*.wma,*.wmv,*.aac,*.avi,*.bmp,*.csv,*.jar,*.json,*.mpeg,*.oga,*.ogv,*.ogx,*.png,*.swf,*.ttf,*.weba,*.webm,*.webp,*.3gp,*.3g2,*.exe,*.sh,*.xls,*.xlsx,*.mov,*.mpg,*.m4v,*.flv,*.dbf,*.msi
ignore_case = on
recursive = on
verbose = off
wait = 2
random_wait = on
adjust_extension = on
restrict_file_names = unix
warc_max_size = 1G
warc_cdx = on
delete_after = on
user_agent = test-agent