#!/usr/bin/env python

'''
    This program is free software; you can redistribute it and/or modify
    it under the terms of the Revised BSD License.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    Revised BSD License for more details.

    Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
    Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski

    $FileInfo: pywwwget.py - Last Update: 10/22/2024 Ver. 2.1.2 RC 1 - Author: cooldude2k $
'''

from __future__ import absolute_import, division, print_function, unicode_literals, generators, with_statement, nested_scopes
import re
import os
import sys
import hashlib
import shutil
import platform
import tempfile
import urllib
import zlib
import bz2
import time
import argparse
import subprocess
import socket
import email.utils
import datetime
import time
import logging as log
from ftplib import FTP, FTP_TLS
from base64 import b64encode
try:
    from cgi import parse_qsl
except ImportError:
    try:
        from urlparse import parse_qsl
    except ImportError:
        from urllib.parse import parse_qsl
except (DeprecationWarning, TypeError):
    try:
        from urlparse import parse_qsl
    except ImportError:
        from urllib.parse import parse_qsl
haverequests = False
try:
    import requests
    haverequests = True
except ImportError:
    haverequests = False
havemechanize = False
try:
    import mechanize
    havemechanize = True
except ImportError:
    havemechanize = False
havepycurl = False
try:
    import pycurl
    havepycurl = True
except ImportError:
    havepycurl = False
haveparamiko = False
try:
    import paramiko
    haveparamiko = True
except ImportError:
    haveparamiko = False
havepysftp = False
try:
    import pysftp
    havepysftp = True
except ImportError:
    havepysftp = False
haveurllib3 = False
try:
    import urllib3
    haveurllib3 = True
except ImportError:
    haveurllib3 = False
havehttplib2 = False
try:
    from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout
    havehttplib2 = True
except ImportError:
    havehttplib2 = False
havehttpx = False
try:
    import httpx
    havehttpx = True
except ImportError:
    havehttpx = False
havehttpcore = False
try:
    import httpcore
    havehttpcore = True
except ImportError:
    havehttpcore = False
haveaiohttp = False
try:
    import aiohttp
    haveaiohttp = True
except ImportError:
    haveaiohttp = False
havebrotli = False
try:
    import brotli
    havebrotli = True
except ImportError:
    havebrotli = False
havezstd = False
try:
    import zstandard
    havezstd = True
except ImportError:
    havezstd = False
havelzma = False
try:
    import lzma
    havelzma = True
except ImportError:
    havelzma = False
if(sys.version[0] == "2"):
    try:
        from io import StringIO, BytesIO
    except ImportError:
        try:
            from cStringIO import StringIO
            from cStringIO import StringIO as BytesIO
        except ImportError:
            from StringIO import StringIO
            from StringIO import StringIO as BytesIO
    # From http://python-future.org/compatible_idioms.html
    from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin
    from urllib import urlencode
    from urllib import urlopen as urlopenalt
    from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor
    import urlparse
    import cookielib
    from httplib import HTTPConnection, HTTPSConnection
if(sys.version[0] >= "3"):
    from io import StringIO, BytesIO
    # From http://python-future.org/compatible_idioms.html
    from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode
    from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor
    from urllib.error import HTTPError, URLError
    import urllib.parse as urlparse
    import http.cookiejar as cookielib
    from http.client import HTTPConnection, HTTPSConnection

__program_name__ = "PyWWW-Get"
__program_alt_name__ = "PyWWWGet"
__program_small_name__ = "wwwget"
__project__ = __program_name__
__project_url__ = "https://github.com/GameMaker2k/PyWWW-Get"
__version_info__ = (2, 1, 2, "RC 1", 1)
__version_date_info__ = (2024, 10, 22, "RC 1", 1)
__version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[
    1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2)
__revision__ = __version_info__[3]
__revision_id__ = "$Id: 1bb0844a3caf29e0aaf9379f076ae1b4eb46529f $"
if(__version_info__[4] is not None):
    __version_date_plusrc__ = __version_date__ + \
        "-"+str(__version_date_info__[4])
if(__version_info__[4] is None):
    __version_date_plusrc__ = __version_date__
if(__version_info__[3] is not None):
    __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(
        __version_info__[2])+" "+str(__version_info__[3])
if(__version_info__[3] is None):
    __version__ = str(
        __version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])

tmpfileprefix = "py" + \
    str(sys.version_info[0])+__program_small_name__ + \
    str(__version_info__[0])+"-"
tmpfilesuffix = "-"
pytempdir = tempfile.gettempdir()

PyBitness = platform.architecture()
if(PyBitness == "32bit" or PyBitness == "32"):
    PyBitness = "32"
elif(PyBitness == "64bit" or PyBitness == "64"):
    PyBitness = "64"
else:
    PyBitness = "32"

compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2']
if(havebrotli):
    compression_supported_list.append('br')
if(havezstd):
    compression_supported_list.append('zstd')
if(havelzma):
    compression_supported_list.append('lzma')
    compression_supported_list.append('xz')
compression_supported = ', '.join(compression_supported_list)

geturls_cj = cookielib.CookieJar()
windowsNT4_ua_string = "Windows NT 4.0"
windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                       'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"}
windows2k_ua_string = "Windows NT 5.0"
windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                      'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"}
windowsXP_ua_string = "Windows NT 5.1"
windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                      'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"}
windowsXP64_ua_string = "Windows NT 5.2; Win64; x64"
windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                        'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"}
windows7_ua_string = "Windows NT 6.1; Win64; x64"
windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                     'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"}
windows8_ua_string = "Windows NT 6.2; Win64; x64"
windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                     'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"}
windows81_ua_string = "Windows NT 6.3; Win64; x64"
windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                      'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"}
windows10_ua_string = "Windows NT 10.0; Win64; x64"
windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                      'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"}
windows11_ua_string = "Windows NT 11.0; Win64; x64"
windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86",
                      'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"}
geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    "; rv:109.0) Gecko/20100101 Firefox/117.0"
geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    "; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17"
geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    ") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    ") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36"
geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    "; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1"
geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    ") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0"
geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    ") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48"
geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 (" + \
    windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko"
geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string + \
    ") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31"
geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(
    proname=__project__, prover=__version__, prourl=__project_url__)
if(platform.python_implementation() != ""):
    py_implementation = platform.python_implementation()
if(platform.python_implementation() == ""):
    py_implementation = "Python"
geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system(
)+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__)
geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)"
geturls_ua = geturls_ua_firefox_windows7
geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                    'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                      'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7",
                                   'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"}
geturls_headers_chrome_windows7.update(windows7_ua_addon)
geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                     'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"}
geturls_headers_chromium_windows7.update(windows7_ua_addon)
geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                     'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7",
                                  'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"}
geturls_headers_opera_windows7.update(windows7_ua_addon)
geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7",
                                    'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"}
geturls_headers_vivaldi_windows7.update(windows7_ua_addon)
geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language':
                                              "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7",
                                           'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon)
geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close",
                                   'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)}
geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close",
                                       'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)}
geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                    'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6",
                                        'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}
geturls_headers = geturls_headers_firefox_windows7
geturls_download_sleep = 0


def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
    if(outtype == "print" and dbgenable):
        print(dbgtxt)
        return True
    elif(outtype == "log" and dbgenable):
        logging.info(dbgtxt)
        return True
    elif(outtype == "warning" and dbgenable):
        logging.warning(dbgtxt)
        return True
    elif(outtype == "error" and dbgenable):
        logging.error(dbgtxt)
        return True
    elif(outtype == "critical" and dbgenable):
        logging.critical(dbgtxt)
        return True
    elif(outtype == "exception" and dbgenable):
        logging.exception(dbgtxt)
        return True
    elif(outtype == "logalt" and dbgenable):
        logging.log(dgblevel, dbgtxt)
        return True
    elif(outtype == "debug" and dbgenable):
        logging.debug(dbgtxt)
        return True
    elif(not dbgenable):
        return True
    else:
        return False
    return False


def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
    dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel)
    if(not dbgout):
        return False
    return dbgtxt


def add_url_param(url, **params):
    n = 3
    parts = list(urlparse.urlsplit(url))
    d = dict(parse_qsl(parts[n]))  # use cgi.parse_qs for list values
    d.update(params)
    parts[n] = urlencode(d)
    return urlparse.urlunsplit(parts)


os.environ["PATH"] = os.environ["PATH"] + os.pathsep + \
    os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd()


def which_exec(execfile):
    for path in os.environ["PATH"].split(":"):
        if os.path.exists(path + "/" + execfile):
            return path + "/" + execfile


def listize(varlist):
    il = 0
    ix = len(varlist)
    ilx = 1
    newlistreg = {}
    newlistrev = {}
    newlistfull = {}
    while(il < ix):
        newlistreg.update({ilx: varlist[il]})
        newlistrev.update({varlist[il]: ilx})
        ilx = ilx + 1
        il = il + 1
    newlistfull = {1: newlistreg, 2: newlistrev,
                   'reg': newlistreg, 'rev': newlistrev}
    return newlistfull


def twolistize(varlist):
    il = 0
    ix = len(varlist)
    ilx = 1
    newlistnamereg = {}
    newlistnamerev = {}
    newlistdescreg = {}
    newlistdescrev = {}
    newlistfull = {}
    while(il < ix):
        newlistnamereg.update({ilx: varlist[il][0].strip()})
        newlistnamerev.update({varlist[il][0].strip(): ilx})
        newlistdescreg.update({ilx: varlist[il][1].strip()})
        newlistdescrev.update({varlist[il][1].strip(): ilx})
        ilx = ilx + 1
        il = il + 1
    newlistnametmp = {1: newlistnamereg, 2: newlistnamerev,
                      'reg': newlistnamereg, 'rev': newlistnamerev}
    newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev,
                      'reg': newlistdescreg, 'rev': newlistdescrev}
    newlistfull = {1: newlistnametmp, 2: newlistdesctmp,
                   'name': newlistnametmp, 'desc': newlistdesctmp}
    return newlistfull


def arglistize(proexec, *varlist):
    il = 0
    ix = len(varlist)
    ilx = 1
    newarglist = [proexec]
    while(il < ix):
        if varlist[il][0] is not None:
            newarglist.append(varlist[il][0])
        if varlist[il][1] is not None:
            newarglist.append(varlist[il][1])
        il = il + 1
    return newarglist


def fix_header_names(header_dict):
    if(sys.version[0] == "2"):
        header_dict = {k.title(): v for k, v in header_dict.iteritems()}
    if(sys.version[0] >= "3"):
        header_dict = {k.title(): v for k, v in header_dict.items()}
    return header_dict

# hms_string by ArcGIS Python Recipes
# https://arcpy.wordpress.com/2012/04/20/146/


def hms_string(sec_elapsed):
    h = int(sec_elapsed / (60 * 60))
    m = int((sec_elapsed % (60 * 60)) / 60)
    s = sec_elapsed % 60.0
    return "{}:{:>02}:{:>05.2f}".format(h, m, s)

# get_readable_size by Lipis
# http://stackoverflow.com/posts/14998888/revisions


def get_readable_size(bytes, precision=1, unit="IEC"):
    unit = unit.upper()
    if(unit != "IEC" and unit != "SI"):
        unit = "IEC"
    if(unit == "IEC"):
        units = [" B", " KiB", " MiB", " GiB", " TiB", " PiB", " EiB", " ZiB"]
        unitswos = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB"]
        unitsize = 1024.0
    if(unit == "SI"):
        units = [" B", " kB", " MB", " GB", " TB", " PB", " EB", " ZB"]
        unitswos = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB"]
        unitsize = 1000.0
    return_val = {}
    orgbytes = bytes
    for unit in units:
        if abs(bytes) < unitsize:
            strformat = "%3."+str(precision)+"f%s"
            pre_return_val = (strformat % (bytes, unit))
            pre_return_val = re.sub(
                r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val)
            pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val)
            alt_return_val = pre_return_val.split()
            return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val,
                          'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
            return return_val
        bytes /= unitsize
    strformat = "%."+str(precision)+"f%s"
    pre_return_val = (strformat % (bytes, "YiB"))
    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val)
    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val)
    alt_return_val = pre_return_val.split()
    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val,
                  'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
    return return_val


def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
    unit = unit.upper()
    usehashtypes = usehashtypes.lower()
    getfilesize = os.path.getsize(infile)
    return_val = get_readable_size(getfilesize, precision, unit)
    if(usehashes):
        hashtypelist = usehashtypes.split(",")
        openfile = open(infile, "rb")
        filecontents = openfile.read()
        openfile.close()
        listnumcount = 0
        listnumend = len(hashtypelist)
        while(listnumcount < listnumend):
            hashtypelistlow = hashtypelist[listnumcount].strip()
            hashtypelistup = hashtypelistlow.upper()
            filehash = hashlib.new(hashtypelistup)
            filehash.update(filecontents)
            filegethash = filehash.hexdigest()
            return_val.update({hashtypelistup: filegethash})
            listnumcount += 1
    return return_val


def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
    unit = unit.upper()
    usehashtypes = usehashtypes.lower()
    getfilesize = len(instring)
    return_val = get_readable_size(getfilesize, precision, unit)
    if(usehashes):
        hashtypelist = usehashtypes.split(",")
        listnumcount = 0
        listnumend = len(hashtypelist)
        while(listnumcount < listnumend):
            hashtypelistlow = hashtypelist[listnumcount].strip()
            hashtypelistup = hashtypelistlow.upper()
            filehash = hashlib.new(hashtypelistup)
            if(sys.version[0] == "2"):
                filehash.update(instring)
            if(sys.version[0] >= "3"):
                filehash.update(instring.encode('utf-8'))
            filegethash = filehash.hexdigest()
            return_val.update({hashtypelistup: filegethash})
            listnumcount += 1
    return return_val


def http_status_to_reason(code):
    reasons = {
        100: 'Continue',
        101: 'Switching Protocols',
        102: 'Processing',
        200: 'OK',
        201: 'Created',
        202: 'Accepted',
        203: 'Non-Authoritative Information',
        204: 'No Content',
        205: 'Reset Content',
        206: 'Partial Content',
        207: 'Multi-Status',
        208: 'Already Reported',
        226: 'IM Used',
        300: 'Multiple Choices',
        301: 'Moved Permanently',
        302: 'Found',
        303: 'See Other',
        304: 'Not Modified',
        305: 'Use Proxy',
        307: 'Temporary Redirect',
        308: 'Permanent Redirect',
        400: 'Bad Request',
        401: 'Unauthorized',
        402: 'Payment Required',
        403: 'Forbidden',
        404: 'Not Found',
        405: 'Method Not Allowed',
        406: 'Not Acceptable',
        407: 'Proxy Authentication Required',
        408: 'Request Timeout',
        409: 'Conflict',
        410: 'Gone',
        411: 'Length Required',
        412: 'Precondition Failed',
        413: 'Payload Too Large',
        414: 'URI Too Long',
        415: 'Unsupported Media Type',
        416: 'Range Not Satisfiable',
        417: 'Expectation Failed',
        421: 'Misdirected Request',
        422: 'Unprocessable Entity',
        423: 'Locked',
        424: 'Failed Dependency',
        426: 'Upgrade Required',
        428: 'Precondition Required',
        429: 'Too Many Requests',
        431: 'Request Header Fields Too Large',
        451: 'Unavailable For Legal Reasons',
        500: 'Internal Server Error',
        501: 'Not Implemented',
        502: 'Bad Gateway',
        503: 'Service Unavailable',
        504: 'Gateway Timeout',
        505: 'HTTP Version Not Supported',
        506: 'Variant Also Negotiates',
        507: 'Insufficient Storage',
        508: 'Loop Detected',
        510: 'Not Extended',
        511: 'Network Authentication Required'
    }
    return reasons.get(code, 'Unknown Status Code')


def ftp_status_to_reason(code):
    reasons = {
        110: 'Restart marker reply',
        120: 'Service ready in nnn minutes',
        125: 'Data connection already open; transfer starting',
        150: 'File status okay; about to open data connection',
        200: 'Command okay',
        202: 'Command not implemented, superfluous at this site',
        211: 'System status, or system help reply',
        212: 'Directory status',
        213: 'File status',
        214: 'Help message',
        215: 'NAME system type',
        220: 'Service ready for new user',
        221: 'Service closing control connection',
        225: 'Data connection open; no transfer in progress',
        226: 'Closing data connection',
        227: 'Entering Passive Mode',
        230: 'User logged in, proceed',
        250: 'Requested file action okay, completed',
        257: '"PATHNAME" created',
        331: 'User name okay, need password',
        332: 'Need account for login',
        350: 'Requested file action pending further information',
        421: 'Service not available, closing control connection',
        425: 'Can\'t open data connection',
        426: 'Connection closed; transfer aborted',
        450: 'Requested file action not taken',
        451: 'Requested action aborted. Local error in processing',
        452: 'Requested action not taken. Insufficient storage space in system',
        500: 'Syntax error, command unrecognized',
        501: 'Syntax error in parameters or arguments',
        502: 'Command not implemented',
        503: 'Bad sequence of commands',
        504: 'Command not implemented for that parameter',
        530: 'Not logged in',
        532: 'Need account for storing files',
        550: 'Requested action not taken. File unavailable',
        551: 'Requested action aborted. Page type unknown',
        552: 'Requested file action aborted. Exceeded storage allocation',
        553: 'Requested action not taken. File name not allowed'
    }
    return reasons.get(code, 'Unknown Status Code')


def sftp_status_to_reason(code):
    reasons = {
        0: 'SSH_FX_OK',
        1: 'SSH_FX_EOF',
        2: 'SSH_FX_NO_SUCH_FILE',
        3: 'SSH_FX_PERMISSION_DENIED',
        4: 'SSH_FX_FAILURE',
        5: 'SSH_FX_BAD_MESSAGE',
        6: 'SSH_FX_NO_CONNECTION',
        7: 'SSH_FX_CONNECTION_LOST',
        8: 'SSH_FX_OP_UNSUPPORTED'
    }
    return reasons.get(code, 'Unknown Status Code')


def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
    if isinstance(headers, dict):
        returnval = []
        if(sys.version[0] == "2"):
            for headkey, headvalue in headers.iteritems():
                returnval.append((headkey, headvalue))
        if(sys.version[0] >= "3"):
            for headkey, headvalue in headers.items():
                returnval.append((headkey, headvalue))
    elif isinstance(headers, list):
        returnval = headers
    else:
        returnval = False
    return returnval


def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
    if isinstance(headers, dict):
        returnval = []
        if(sys.version[0] == "2"):
            for headkey, headvalue in headers.iteritems():
                returnval.append(headkey+": "+headvalue)
        if(sys.version[0] >= "3"):
            for headkey, headvalue in headers.items():
                returnval.append(headkey+": "+headvalue)
    elif isinstance(headers, list):
        returnval = headers
    else:
        returnval = False
    return returnval


def make_http_headers_from_pycurl_to_dict(headers):
    header_dict = {}
    headers = headers.strip().split('\r\n')
    for header in headers:
        parts = header.split(': ', 1)
        if(len(parts) == 2):
            key, value = parts
            header_dict[key.title()] = value
    return header_dict


def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
    if isinstance(headers, list):
        returnval = {}
        mli = 0
        mlil = len(headers)
        while(mli < mlil):
            returnval.update({headers[mli][0]: headers[mli][1]})
            mli = mli + 1
    elif isinstance(headers, dict):
        returnval = headers
    else:
        returnval = False
    return returnval


def get_httplib_support(checkvalue=None):
    global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp
    returnval = []
    returnval.append("ftp")
    returnval.append("httplib")
    if(havehttplib2):
        returnval.append("httplib2")
    returnval.append("urllib")
    if(haveurllib3):
        returnval.append("urllib3")
        returnval.append("request3")
    returnval.append("request")
    if(haverequests):
        returnval.append("requests")
    if(haveaiohttp):
        returnval.append("aiohttp")
    if(havehttpx):
        returnval.append("httpx")
        returnval.append("httpx2")
    if(havemechanize):
        returnval.append("mechanize")
    if(havepycurl):
        returnval.append("pycurl")
        if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
            returnval.append("pycurl2")
        if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
            returnval.append("pycurl3")
    if(haveparamiko):
        returnval.append("sftp")
    if(havepysftp):
        returnval.append("pysftp")
    if(not checkvalue is None):
        if(checkvalue == "urllib1" or checkvalue == "urllib2"):
            checkvalue = "urllib"
        if(checkvalue == "httplib1"):
            checkvalue = "httplib"
        if(checkvalue in returnval):
            returnval = True
        else:
            returnval = False
    return returnval


def check_httplib_support(checkvalue="urllib"):
    if(checkvalue == "urllib1" or checkvalue == "urllib2"):
        checkvalue = "urllib"
    if(checkvalue == "httplib1"):
        checkvalue = "httplib"
    returnval = get_httplib_support(checkvalue)
    return returnval


def get_httplib_support_list():
    returnval = get_httplib_support(None)
    return returnval


def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
    global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp
    if(sleep < 0):
        sleep = geturls_download_sleep
    if(timeout <= 0):
        timeout = 10
    if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"):
        httplibuse = "urllib"
    if(httplibuse == "httplib1"):
        httplibuse = "httplib"
    if(not haverequests and httplibuse == "requests"):
        httplibuse = "urllib"
    if(not haveaiohttp and httplibuse == "aiohttp"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx2"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore2"):
        httplibuse = "urllib"
    if(not havemechanize and httplibuse == "mechanize"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl2"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havepycurl and httplibuse == "pycurl3"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl2"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havehttplib2 and httplibuse == "httplib2"):
        httplibuse = "httplib"
    if(not haveparamiko and httplibuse == "sftp"):
        httplibuse = "ftp"
    if(not havepysftp and httplibuse == "pysftp"):
        httplibuse = "ftp"
    urlparts = urlparse.urlparse(httpurl)
    if(isinstance(httpheaders, list)):
        httpheaders = make_http_headers_from_list_to_dict(httpheaders)
    httpheaders = fix_header_names(httpheaders)
    if(httpuseragent is not None):
        if('User-Agent' in httpheaders):
            httpheaders['User-Agent'] = httpuseragent
        else:
            httpuseragent.update({'User-Agent': httpuseragent})
    if(httpreferer is not None):
        if('Referer' in httpheaders):
            httpheaders['Referer'] = httpreferer
        else:
            httpuseragent.update({'Referer': httpreferer})
    if(urlparts.username is not None or urlparts.password is not None):
        if(sys.version[0] == "2"):
            inurlencode = b64encode(
                str(urlparts.username+":"+urlparts.password))
        if(sys.version[0] >= "3"):
            inurlencode = b64encode(
                str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8")
        httpheaders.update({'Authorization': "Basic "+inurlencode})
    geturls_opener = build_opener(HTTPCookieProcessor(httpcookie))
    if(httplibuse == "urllib" or httplibuse == "mechanize"):
        if(isinstance(httpheaders, dict)):
            httpheaders = make_http_headers_from_dict_to_list(httpheaders)
    if(httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3"):
        if(isinstance(httpheaders, dict)):
            httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders)
    geturls_opener.addheaders = httpheaders
    time.sleep(sleep)
    if(postdata is not None and not isinstance(postdata, dict)):
        postdata = urlencode(postdata)
    if(httplibuse == "urllib" or httplibuse == "request"):
        geturls_request = Request(httpurl)
        try:
            if(httpmethod == "GET"):
                geturls_text = geturls_opener.open(geturls_request)
            elif(httpmethod == "POST"):
                geturls_text = geturls_opener.open(
                    geturls_request, data=postdata)
            else:
                geturls_text = geturls_opener.open(geturls_request)
        except HTTPError as geturls_text_error:
            geturls_text = geturls_text_error
            log.info("Error With URL "+httpurl)
        except URLError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.getcode()
        try:
            httpcodereason = geturls_text.reason
        except AttributeError:
            httpcodereason = http_status_to_reason(geturls_text.getcode())
        try:
            httpversionout = geturls_text.version
        except AttributeError:
            httpversionout = "1.1"
        httpmethodout = geturls_request.get_method()
        httpurlout = geturls_text.geturl()
        httpheaderout = geturls_text.info()
        httpheadersentout = httpheaders
    elif(httplibuse == "httplib"):
        if(urlparts[0] == "http"):
            httpconn = HTTPConnection(urlparts[1], timeout=timeout)
        elif(urlparts[0] == "https"):
            httpconn = HTTPSConnection(urlparts[1], timeout=timeout)
        else:
            return False
        if(postdata is not None and not isinstance(postdata, dict)):
            postdata = urlencode(postdata)
        try:
            if(httpmethod == "GET"):
                httpconn.request("GET", urlparts[2], headers=httpheaders)
            elif(httpmethod == "POST"):
                httpconn.request(
                    "GET", urlparts[2], body=postdata, headers=httpheaders)
            else:
                httpconn.request("GET", urlparts[2], headers=httpheaders)
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except BlockingIOError:
            log.info("Error With URL "+httpurl)
            return False
        geturls_text = httpconn.getresponse()
        httpcodeout = geturls_text.status
        httpcodereason = geturls_text.reason
        if(geturls_text.version == "10"):
            httpversionout = "1.0"
        else:
            httpversionout = "1.1"
        httpmethodout = geturls_text._method
        httpurlout = httpurl
        httpheaderout = geturls_text.getheaders()
        httpheadersentout = httpheaders
    elif(httplibuse == "httplib2"):
        if(urlparts[0] == "http"):
            httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout)
        elif(urlparts[0] == "https"):
            httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout)
        else:
            return False
        if(postdata is not None and not isinstance(postdata, dict)):
            postdata = urlencode(postdata)
        try:
            if(httpmethod == "GET"):
                httpconn.request("GET", urlparts[2], headers=httpheaders)
            elif(httpmethod == "POST"):
                httpconn.request(
                    "GET", urlparts[2], body=postdata, headers=httpheaders)
            else:
                httpconn.request("GET", urlparts[2], headers=httpheaders)
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except BlockingIOError:
            log.info("Error With URL "+httpurl)
            return False
        geturls_text = httpconn.getresponse()
        httpcodeout = geturls_text.status
        httpcodereason = geturls_text.reason
        if(geturls_text.version == "10"):
            httpversionout = "1.0"
        else:
            httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = httpurl
        httpheaderout = geturls_text.getheaders()
        httpheadersentout = httpheaders
    elif(httplibuse == "urllib3" or httplibuse == "request3"):
        timeout = urllib3.util.Timeout(connect=timeout, read=timeout)
        urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout)
        try:
            if(httpmethod == "GET"):
                geturls_text = urllib_pool.request(
                    "GET", httpurl, headers=httpheaders, preload_content=False)
            elif(httpmethod == "POST"):
                geturls_text = urllib_pool.request(
                    "POST", httpurl, body=postdata, headers=httpheaders, preload_content=False)
            else:
                geturls_text = urllib_pool.request(
                    "GET", httpurl, headers=httpheaders, preload_content=False)
        except urllib3.exceptions.ConnectTimeoutError:
            log.info("Error With URL "+httpurl)
            return False
        except urllib3.exceptions.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except urllib3.exceptions.MaxRetryError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except ValueError:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status
        httpcodereason = geturls_text.reason
        if(geturls_text.version == "10"):
            httpversionout = "1.0"
        else:
            httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = geturls_text.geturl()
        httpheaderout = geturls_text.info()
        httpheadersentout = httpheaders
    elif(httplibuse == "requests"):
        try:
            reqsession = requests.Session()
            if(httpmethod == "GET"):
                geturls_text = reqsession.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
            elif(httpmethod == "POST"):
                geturls_text = reqsession.post(
                    httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie)
            else:
                geturls_text = reqsession.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
        except requests.exceptions.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except requests.exceptions.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status_code
        httpcodereason = geturls_text.reason
        if(geturls_text.raw.version == "10"):
            httpversionout = "1.0"
        else:
            httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = geturls_text.url
        httpheaderout = geturls_text.headers
        httpheadersentout = geturls_text.request.headers
    elif(httplibuse == "aiohttp"):
        try:
            reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders,
                                               timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize)
            if(httpmethod == "GET"):
                geturls_text = reqsession.get(httpurl)
            elif(httpmethod == "POST"):
                geturls_text = reqsession.post(httpurl, data=postdata)
            else:
                geturls_text = reqsession.get(httpurl)
        except aiohttp.exceptions.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except aiohttp.exceptions.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status
        httpcodereason = geturls_text.reason
        httpversionout = geturls_text.version
        httpmethodout = geturls_text.method
        httpurlout = geturls_text.url
        httpheaderout = geturls_text.headers
        httpheadersentout = geturls_text.request_info.headers
    elif(httplibuse == "httpx"):
        try:
            if(httpmethod == "GET"):
                httpx_pool = httpx.Client(
                    http1=True, http2=False, trust_env=True)
                geturls_text = httpx_pool.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
            elif(httpmethod == "POST"):
                httpx_pool = httpx.Client(
                    http1=True, http2=False, trust_env=True)
                geturls_text = httpx_pool.post(
                    httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie)
            else:
                httpx_pool = httpx.Client(
                    http1=True, http2=False, trust_env=True)
                geturls_text = httpx_pool.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
        except httpx.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except httpx.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status_code
        try:
            httpcodereason = geturls_text.reason_phrase
        except:
            httpcodereason = http_status_to_reason(geturls_text.status_code)
        httpversionout = geturls_text.http_version
        httpmethodout = httpmethod
        httpurlout = str(geturls_text.url)
        httpheaderout = geturls_text.headers
        httpheadersentout = geturls_text.request.headers
    elif(httplibuse == "httpx2"):
        try:
            if(httpmethod == "GET"):
                httpx_pool = httpx.Client(
                    http1=True, http2=True, trust_env=True)
                geturls_text = httpx_pool.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
            elif(httpmethod == "POST"):
                httpx_pool = httpx.Client(
                    http1=True, http2=True, trust_env=True)
                geturls_text = httpx_pool.post(
                    httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie)
            else:
                httpx_pool = httpx.Client(
                    http1=True, http2=True, trust_env=True)
                geturls_text = httpx_pool.get(
                    httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie)
        except httpx.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except httpx.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status_code
        try:
            httpcodereason = geturls_text.reason_phrase
        except:
            httpcodereason = http_status_to_reason(geturls_text.status_code)
        httpversionout = geturls_text.http_version
        httpmethodout = httpmethod
        httpurlout = str(geturls_text.url)
        httpheaderout = geturls_text.headers
        httpheadersentout = geturls_text.request.headers
    elif(httplibuse == "httpcore"):
        try:
            if(httpmethod == "GET"):
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=False)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, headers=httpheaders)
            elif(httpmethod == "POST"):
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=False)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, data=postdata, headers=httpheaders)
            else:
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=False)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, headers=httpheaders)
        except httpcore.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except httpcore.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status
        httpcodereason = http_status_to_reason(geturls_text.status)
        httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = str(httpurl)
        httpheaderout = geturls_text.headers
        httpheadersentout = httpheaders
    elif(httplibuse == "httpcore2"):
        try:
            if(httpmethod == "GET"):
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=True)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, headers=httpheaders)
            elif(httpmethod == "POST"):
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=True)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, data=postdata, headers=httpheaders)
            else:
                httpx_pool = httpcore.ConnectionPool(http1=True, http2=True)
                geturls_text = httpx_pool.request(
                    "GET", httpurl, headers=httpheaders)
        except httpcore.ConnectTimeout:
            log.info("Error With URL "+httpurl)
            return False
        except httpcore.ConnectError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.status
        httpcodereason = http_status_to_reason(geturls_text.status)
        httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = str(httpurl)
        httpheaderout = geturls_text.headers
        httpheadersentout = httpheaders
    elif(httplibuse == "mechanize"):
        geturls_opener = mechanize.Browser()
        if(isinstance(httpheaders, dict)):
            httpheaders = make_http_headers_from_dict_to_list(httpheaders)
        time.sleep(sleep)
        geturls_opener.addheaders = httpheaders
        geturls_opener.set_cookiejar(httpcookie)
        geturls_opener.set_handle_robots(False)
        if(postdata is not None and not isinstance(postdata, dict)):
            postdata = urlencode(postdata)
        try:
            if(httpmethod == "GET"):
                geturls_text = geturls_opener.open(httpurl)
            elif(httpmethod == "POST"):
                geturls_text = geturls_opener.open(httpurl, data=postdata)
            else:
                geturls_text = geturls_opener.open(httpurl)
        except mechanize.HTTPError as geturls_text_error:
            geturls_text = geturls_text_error
            log.info("Error With URL "+httpurl)
        except URLError:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.code
        httpcodereason = geturls_text.msg
        httpversionout = "1.1"
        httpmethodout = httpmethod
        httpurlout = geturls_text.geturl()
        httpheaderout = geturls_text.info()
        reqhead = geturls_opener.request
        httpheadersentout = reqhead.header_items()
    elif(httplibuse == "pycurl"):
        retrieved_body = BytesIO()
        retrieved_headers = BytesIO()
        try:
            if(httpmethod == "GET"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_1_1)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            elif(httpmethod == "POST"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_1_1)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.setopt(geturls_text.POST, True)
                geturls_text.setopt(geturls_text.POSTFIELDS, postdata)
                geturls_text.perform()
            else:
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_1_1)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            retrieved_headers.seek(0)
            if(sys.version[0] == "2"):
                pycurlhead = retrieved_headers.read()
            if(sys.version[0] >= "3"):
                pycurlhead = retrieved_headers.read().decode('UTF-8')
            pyhttpverinfo = re.findall(
                r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0]
            pycurlheadersout = make_http_headers_from_pycurl_to_dict(
                pycurlhead)
            retrieved_body.seek(0)
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except ValueError:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE)
        httpcodereason = http_status_to_reason(
            geturls_text.getinfo(geturls_text.HTTP_CODE))
        httpversionout = pyhttpverinfo[0]
        httpmethodout = httpmethod
        httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL)
        httpheaderout = pycurlheadersout
        httpheadersentout = httpheaders
    elif(httplibuse == "pycurl2"):
        retrieved_body = BytesIO()
        retrieved_headers = BytesIO()
        try:
            if(httpmethod == "GET"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_2_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            elif(httpmethod == "POST"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_2_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.setopt(geturls_text.POST, True)
                geturls_text.setopt(geturls_text.POSTFIELDS, postdata)
                geturls_text.perform()
            else:
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_2_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            retrieved_headers.seek(0)
            if(sys.version[0] == "2"):
                pycurlhead = retrieved_headers.read()
            if(sys.version[0] >= "3"):
                pycurlhead = retrieved_headers.read().decode('UTF-8')
            pyhttpverinfo = re.findall(
                r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0]
            pycurlheadersout = make_http_headers_from_pycurl_to_dict(
                pycurlhead)
            retrieved_body.seek(0)
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except ValueError:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE)
        httpcodereason = http_status_to_reason(
            geturls_text.getinfo(geturls_text.HTTP_CODE))
        httpversionout = pyhttpverinfo[0]
        httpmethodout = httpmethod
        httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL)
        httpheaderout = pycurlheadersout
        httpheadersentout = httpheaders
    elif(httplibuse == "pycurl3"):
        retrieved_body = BytesIO()
        retrieved_headers = BytesIO()
        try:
            if(httpmethod == "GET"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_3_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            elif(httpmethod == "POST"):
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_3_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.setopt(geturls_text.POST, True)
                geturls_text.setopt(geturls_text.POSTFIELDS, postdata)
                geturls_text.perform()
            else:
                geturls_text = pycurl.Curl()
                geturls_text.setopt(geturls_text.HTTP_VERSION,
                                    geturls_text.CURL_HTTP_VERSION_3_0)
                geturls_text.setopt(geturls_text.URL, httpurl)
                geturls_text.setopt(
                    geturls_text.WRITEFUNCTION, retrieved_body.write)
                geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders)
                geturls_text.setopt(
                    geturls_text.HEADERFUNCTION, retrieved_headers.write)
                geturls_text.setopt(geturls_text.FOLLOWLOCATION, True)
                geturls_text.setopt(geturls_text.TIMEOUT, timeout)
                geturls_text.perform()
            retrieved_headers.seek(0)
            if(sys.version[0] == "2"):
                pycurlhead = retrieved_headers.read()
            if(sys.version[0] >= "3"):
                pycurlhead = retrieved_headers.read().decode('UTF-8')
            pyhttpverinfo = re.findall(
                r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0]
            pycurlheadersout = make_http_headers_from_pycurl_to_dict(
                pycurlhead)
            retrieved_body.seek(0)
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except ValueError:
            log.info("Error With URL "+httpurl)
            return False
        httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE)
        httpcodereason = http_status_to_reason(
            geturls_text.getinfo(geturls_text.HTTP_CODE))
        httpversionout = pyhttpverinfo[0]
        httpmethodout = httpmethod
        httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL)
        httpheaderout = pycurlheadersout
        httpheadersentout = httpheaders
    elif(httplibuse == "ftp"):
        geturls_text = download_file_from_ftp_file(httpurl)
        if(not geturls_text):
            return False
        downloadsize = None
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(
            fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None}
        geturls_text.close()
    elif(httplibuse == "sftp"):
        geturls_text = download_file_from_sftp_file(httpurl)
        if(not geturls_text):
            return False
        downloadsize = None
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(
            fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None}
        geturls_text.close()
        return returnval
    elif(httplibuse == "pysftp"):
        geturls_text = download_file_from_pysftp_file(httpurl)
        if(not geturls_text):
            return False
        downloadsize = None
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(
            fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None}
        geturls_text.close()
        return returnval
    else:
        returnval = False
    if(isinstance(httpheaderout, list) and (httplibuse != "pycurl" and httplibuse != "pycurl2" and httplibuse != "pycurl3")):
        httpheaderout = dict(
            make_http_headers_from_list_to_dict(httpheaderout))
    if(isinstance(httpheaderout, list) and (httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3")):
        httpheaderout = dict(make_http_headers_from_pycurl_to_dict(
            "\r\n".join(httpheaderout)))
    if(sys.version[0] == "2"):
        try:
            prehttpheaderout = httpheaderout
            httpheaderkeys = httpheaderout.keys()
            imax = len(httpheaderkeys)
            ic = 0
            httpheaderout = {}
            while(ic < imax):
                httpheaderout.update(
                    {httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]})
                ic += 1
        except AttributeError:
            pass
    httpheaderout = fix_header_names(httpheaderout)
    if(isinstance(httpheadersentout, list) and (httplibuse != "pycurl" and httplibuse != "pycurl2" and httplibuse != "pycurl3")):
        httpheadersentout = dict(
            make_http_headers_from_list_to_dict(httpheadersentout))
    if(isinstance(httpheadersentout, list) and (httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3")):
        httpheadersentout = dict(make_http_headers_from_pycurl_to_dict(
            "\r\n".join(httpheadersentout)))
    httpheadersentout = fix_header_names(httpheadersentout)
    log.info("Downloading URL "+httpurl)
    if(httplibuse == "urllib" or httplibuse == "request" or httplibuse == "request3" or httplibuse == "aiohttp" or httplibuse == "httplib" or httplibuse == "httplib2" or httplibuse == "urllib3" or httplibuse == "mechanize"):
        downloadsize = httpheaderout.get('Content-Length')
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        geturls_text.close()
        if(httpheaderout.get("Content-Encoding") == "gzip"):
            try:
                returnval_content = zlib.decompress(
                    returnval_content, 16+zlib.MAX_WBITS)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "deflate"):
            try:
                returnval_content = zlib.decompress(returnval_content)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli):
            try:
                returnval_content = brotli.decompress(returnval_content)
            except brotli.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd):
            try:
                returnval_content = zstandard.decompress(returnval_content)
            except zstandard.error:
                pass
        elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma):
            try:
                returnval_content = lzma.decompress(returnval_content)
            except zstandard.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "bzip2"):
            try:
                returnval_content = bz2.decompress(returnval_content)
            except zstandard.error:
                pass
    elif(httplibuse == "httpx" or httplibuse == "httpx2" or httplibuse == "httpcore" or httplibuse == "httpcore2"):
        downloadsize = httpheaderout.get('Content-Length')
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.read()
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
                break
            strbuf.seek(0)
            returnval_content = strbuf.read()
        geturls_text.close()
        if(httpheaderout.get("Content-Encoding") == "gzip"):
            try:
                returnval_content = zlib.decompress(
                    returnval_content, 16+zlib.MAX_WBITS)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "deflate"):
            try:
                returnval_content = zlib.decompress(returnval_content)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli):
            try:
                returnval_content = brotli.decompress(returnval_content)
            except brotli.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd):
            try:
                returnval_content = zstandard.decompress(returnval_content)
            except zstandard.error:
                pass
        elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma):
            try:
                returnval_content = lzma.decompress(returnval_content)
            except zstandard.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "bzip2"):
            try:
                returnval_content = bz2.decompress(returnval_content)
            except zstandard.error:
                pass
    elif(httplibuse == "requests"):
        log.info("Downloading URL "+httpurl)
        downloadsize = httpheaderout.get('Content-Length')
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = geturls_text.raw.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        geturls_text.close()
        if(httpheaderout.get("Content-Encoding") == "gzip"):
            try:
                returnval_content = zlib.decompress(
                    returnval_content, 16+zlib.MAX_WBITS)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "deflate"):
            try:
                returnval_content = zlib.decompress(returnval_content)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli):
            try:
                returnval_content = brotli.decompress(returnval_content)
            except brotli.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd):
            try:
                returnval_content = zstandard.decompress(returnval_content)
            except zstandard.error:
                pass
        elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma):
            try:
                returnval_content = lzma.decompress(returnval_content)
            except zstandard.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "bzip2"):
            try:
                returnval_content = bz2.decompress(returnval_content)
            except zstandard.error:
                pass
    elif(httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3"):
        log.info("Downloading URL "+httpurl)
        downloadsize = httpheaderout.get('Content-Length')
        if(downloadsize is not None):
            downloadsize = int(downloadsize)
        if downloadsize is None:
            downloadsize = 0
        fulldatasize = 0
        prevdownsize = 0
        log.info("Downloading URL "+httpurl)
        with BytesIO() as strbuf:
            while True:
                databytes = retrieved_body.read(buffersize)
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                strbuf.write(databytes)
            strbuf.seek(0)
            returnval_content = strbuf.read()
        geturls_text.close()
        if(httpheaderout.get("Content-Encoding") == "gzip"):
            try:
                returnval_content = zlib.decompress(
                    returnval_content, 16+zlib.MAX_WBITS)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "deflate"):
            try:
                returnval_content = zlib.decompress(returnval_content)
            except zlib.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli):
            try:
                returnval_content = brotli.decompress(returnval_content)
            except brotli.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd):
            try:
                returnval_content = zstandard.decompress(returnval_content)
            except zstandard.error:
                pass
        elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma):
            try:
                returnval_content = lzma.decompress(returnval_content)
            except zstandard.error:
                pass
        elif(httpheaderout.get("Content-Encoding") == "bzip2"):
            try:
                returnval_content = bz2.decompress(returnval_content)
            except zstandard.error:
                pass
    elif(httplibuse == "ftp" or httplibuse == "sftp" or httplibuse == "pysftp"):
        pass
    else:
        returnval = False
    returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(
        fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse}
    return returnval


def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
    if(isinstance(httpurl, list)):
        pass
    elif(isinstance(httpurl, tuple)):
        pass
    elif(isinstance(httpurl, dict)):
        httpurl = httpurl.values()
    else:
        httpurl = [httpurl]
    listsize = len(httpurl)
    listcount = 0
    returnval = []
    while(listcount < listsize):
        ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer,
                                     httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout)
        returnval.append(ouputval)
        listcount += 1
    return returnval


def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp
    exec_time_start = time.time()
    myhash = hashlib.new("sha1")
    if(sys.version[0] == "2"):
        myhash.update(httpurl)
        myhash.update(str(buffersize))
        myhash.update(str(exec_time_start))
    if(sys.version[0] >= "3"):
        myhash.update(httpurl.encode('utf-8'))
        myhash.update(str(buffersize).encode('utf-8'))
        myhash.update(str(exec_time_start).encode('utf-8'))
    newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest())
    if(sleep < 0):
        sleep = geturls_download_sleep
    if(timeout <= 0):
        timeout = 10
    if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"):
        httplibuse = "urllib"
    if(httplibuse == "httplib1"):
        httplibuse = "httplib"
    if(not haverequests and httplibuse == "requests"):
        httplibuse = "urllib"
    if(not haveaiohttp and httplibuse == "aiohttp"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx2"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore2"):
        httplibuse = "urllib"
    if(not havemechanize and httplibuse == "mechanize"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl2"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havepycurl and httplibuse == "pycurl3"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl2"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havehttplib2 and httplibuse == "httplib2"):
        httplibuse = "httplib"
    if(not haveparamiko and httplibuse == "sftp"):
        httplibuse = "ftp"
    if(not haveparamiko and httplibuse == "pysftp"):
        httplibuse = "ftp"
    pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout)
    if(not pretmpfilename):
        return False
    with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
        tmpfilename = f.name
        try:
            os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()),
                     time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())))
        except AttributeError:
            try:
                os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()),
                         time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())))
            except ValueError:
                pass
        except ValueError:
            pass
        returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get(
            'Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')}
        f.write(pretmpfilename.get('Content'))
        f.close()
    exec_time_end = time.time()
    log.info("It took "+hms_string(exec_time_start -
             exec_time_end)+" to download file.")
    returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(
        os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)})
    return returnval


def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    if(isinstance(httpurl, list)):
        pass
    elif(isinstance(httpurl, tuple)):
        pass
    elif(isinstance(httpurl, dict)):
        httpurl = httpurl.values()
    else:
        httpurl = [httpurl]
    listsize = len(httpurl)
    listcount = 0
    returnval = []
    while(listcount < listsize):
        ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer,
                                          httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout)
        returnval.append(ouputval)
        listcount += 1
    return returnval


def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp
    if(sleep < 0):
        sleep = geturls_download_sleep
    if(timeout <= 0):
        timeout = 10
    if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"):
        httplibuse = "urllib"
    if(httplibuse == "httplib1"):
        httplibuse = "httplib"
    if(not haverequests and httplibuse == "requests"):
        httplibuse = "urllib"
    if(not haveaiohttp and httplibuse == "aiohttp"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx"):
        httplibuse = "urllib"
    if(not havehttpx and httplibuse == "httpx2"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore"):
        httplibuse = "urllib"
    if(not havehttpcore and httplibuse == "httpcore2"):
        httplibuse = "urllib"
    if(not havemechanize and httplibuse == "mechanize"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl"):
        httplibuse = "urllib"
    if(not havepycurl and httplibuse == "pycurl2"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havepycurl and httplibuse == "pycurl3"):
        httplibuse = "urllib"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl2"
    if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
        httplibuse = "pycurl"
    if(not havehttplib2 and httplibuse == "httplib2"):
        httplibuse = "httplib"
    if(not haveparamiko and httplibuse == "sftp"):
        httplibuse = "ftp"
    if(not havepysftp and httplibuse == "pysftp"):
        httplibuse = "ftp"
    if(not outfile == "-"):
        outpath = outpath.rstrip(os.path.sep)
        filepath = os.path.realpath(outpath+os.path.sep+outfile)
        if(not os.path.exists(outpath)):
            os.makedirs(outpath)
        if(os.path.exists(outpath) and os.path.isfile(outpath)):
            return False
        if(os.path.exists(filepath) and os.path.isdir(filepath)):
            return False
        pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                                httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout)
        if(not pretmpfilename):
            return False
        tmpfilename = pretmpfilename.get('Filename')
        downloadsize = int(os.path.getsize(tmpfilename))
        fulldatasize = 0
        log.info("Moving file "+tmpfilename+" to "+filepath)
        exec_time_start = time.time()
        shutil.move(tmpfilename, filepath)
        try:
            os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()),
                     time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())))
        except AttributeError:
            try:
                os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()),
                         time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())))
            except ValueError:
                pass
        except ValueError:
            pass
        exec_time_end = time.time()
        log.info("It took "+hms_string(exec_time_start -
                 exec_time_end)+" to move file.")
        if(os.path.exists(tmpfilename)):
            os.remove(tmpfilename)
        returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(
            exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')}
    if(outfile == "-"):
        pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                                httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout)
        tmpfilename = pretmpfilename.get('Filename')
        downloadsize = int(os.path.getsize(tmpfilename))
        fulldatasize = 0
        prevdownsize = 0
        exec_time_start = time.time()
        with open(tmpfilename, 'rb') as ft:
            f = BytesIO()
            while True:
                databytes = ft.read(buffersize[1])
                if not databytes:
                    break
                datasize = len(databytes)
                fulldatasize = datasize + fulldatasize
                percentage = ""
                if(downloadsize > 0):
                    percentage = str("{0:.2f}".format(
                        float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%"
                downloaddiff = fulldatasize - prevdownsize
                log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[
                         'ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix'])
                prevdownsize = fulldatasize
                f.write(databytes)
            f.seek(0)
            fdata = f.getvalue()
            f.close()
            ft.close()
            os.remove(tmpfilename)
            exec_time_end = time.time()
            log.info("It took "+hms_string(exec_time_start -
                     exec_time_end)+" to copy file.")
        returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(
            exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse}
    return returnval


def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    if(isinstance(httpurl, list)):
        pass
    elif(isinstance(httpurl, tuple)):
        pass
    elif(isinstance(httpurl, dict)):
        httpurl = httpurl.values()
    else:
        httpurl = [httpurl]
    listsize = len(httpurl)
    listcount = 0
    returnval = []
    while(listcount < listsize):
        ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer,
                                             httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout)
        returnval.append(ouputval)
        listcount += 1
    return returnval


def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout)
    return returnval


def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer,
                                  httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
    returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer,
                                       httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
    returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie,
                                          httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout)
    return returnval


def download_file_from_ftp_file(url):
    urlparts = urlparse.urlparse(url)
    file_name = os.path.basename(urlparts.path)
    file_dir = os.path.dirname(urlparts.path)
    if(urlparts.username is not None):
        ftp_username = urlparts.username
    else:
        ftp_username = "anonymous"
    if(urlparts.password is not None):
        ftp_password = urlparts.password
    elif(urlparts.password is None and urlparts.username == "anonymous"):
        ftp_password = "anonymous"
    else:
        ftp_password = ""
    if(urlparts.scheme == "ftp"):
        ftp = FTP()
    elif(urlparts.scheme == "ftps"):
        ftp = FTP_TLS()
    else:
        return False
    if(urlparts.scheme == "http" or urlparts.scheme == "https"):
        return False
    ftp_port = urlparts.port
    if(urlparts.port is None):
        ftp_port = 21
    try:
        ftp.connect(urlparts.hostname, ftp_port)
    except socket.gaierror:
        log.info("Error With URL "+httpurl)
        return False
    except socket.timeout:
        log.info("Error With URL "+httpurl)
        return False
    ftp.login(urlparts.username, urlparts.password)
    if(urlparts.scheme == "ftps"):
        ftp.prot_p()
    ftpfile = BytesIO()
    ftp.retrbinary("RETR "+urlparts.path, ftpfile.write)
    #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
    ftp.close()
    ftpfile.seek(0, 0)
    return ftpfile


def download_file_from_ftp_string(url):
    ftpfile = download_file_from_ftp_file(url)
    return ftpfile.read()


def upload_file_to_ftp_file(ftpfile, url):
    urlparts = urlparse.urlparse(url)
    file_name = os.path.basename(urlparts.path)
    file_dir = os.path.dirname(urlparts.path)
    if(urlparts.username is not None):
        ftp_username = urlparts.username
    else:
        ftp_username = "anonymous"
    if(urlparts.password is not None):
        ftp_password = urlparts.password
    elif(urlparts.password is None and urlparts.username == "anonymous"):
        ftp_password = "anonymous"
    else:
        ftp_password = ""
    if(urlparts.scheme == "ftp"):
        ftp = FTP()
    elif(urlparts.scheme == "ftps"):
        ftp = FTP_TLS()
    else:
        return False
    if(urlparts.scheme == "http" or urlparts.scheme == "https"):
        return False
    ftp_port = urlparts.port
    if(urlparts.port is None):
        ftp_port = 21
    try:
        ftp.connect(urlparts.hostname, ftp_port)
    except socket.gaierror:
        log.info("Error With URL "+httpurl)
        return False
    except socket.timeout:
        log.info("Error With URL "+httpurl)
        return False
    ftp.login(urlparts.username, urlparts.password)
    if(urlparts.scheme == "ftps"):
        ftp.prot_p()
    ftp.storbinary("STOR "+urlparts.path, ftpfile)
    ftp.close()
    ftpfile.seek(0, 0)
    return ftpfile


def upload_file_to_ftp_string(ftpstring, url):
    ftpfileo = BytesIO(ftpstring)
    ftpfile = upload_file_to_ftp_file(ftpfileo, url)
    ftpfileo.close()
    return ftpfile


if(haveparamiko):
    def download_file_from_sftp_file(url):
        urlparts = urlparse.urlparse(url)
        file_name = os.path.basename(urlparts.path)
        file_dir = os.path.dirname(urlparts.path)
        if(urlparts.scheme == "http" or urlparts.scheme == "https"):
            return False
        sftp_port = urlparts.port
        if(urlparts.port is None):
            sftp_port = 22
        else:
            sftp_port = urlparts.port
        if(urlparts.username is not None):
            sftp_username = urlparts.username
        else:
            sftp_username = "anonymous"
        if(urlparts.password is not None):
            sftp_password = urlparts.password
        elif(urlparts.password is None and urlparts.username == "anonymous"):
            sftp_password = "anonymous"
        else:
            sftp_password = ""
        if(urlparts.scheme != "sftp"):
            return False
        ssh = paramiko.SSHClient()
        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        try:
            ssh.connect(urlparts.hostname, port=sftp_port,
                        username=urlparts.username, password=urlparts.password)
        except paramiko.ssh_exception.SSHException:
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        sftp = ssh.open_sftp()
        sftpfile = BytesIO()
        sftp.getfo(urlparts.path, sftpfile)
        sftp.close()
        ssh.close()
        sftpfile.seek(0, 0)
        return sftpfile
else:
    def download_file_from_sftp_file(url):
        return False

if(haveparamiko):
    def download_file_from_sftp_string(url):
        sftpfile = download_file_from_sftp_file(url)
        return sftpfile.read()
else:
    def download_file_from_ftp_string(url):
        return False

if(haveparamiko):
    def upload_file_to_sftp_file(sftpfile, url):
        urlparts = urlparse.urlparse(url)
        file_name = os.path.basename(urlparts.path)
        file_dir = os.path.dirname(urlparts.path)
        sftp_port = urlparts.port
        if(urlparts.scheme == "http" or urlparts.scheme == "https"):
            return False
        if(urlparts.port is None):
            sftp_port = 22
        else:
            sftp_port = urlparts.port
        if(urlparts.username is not None):
            sftp_username = urlparts.username
        else:
            sftp_username = "anonymous"
        if(urlparts.password is not None):
            sftp_password = urlparts.password
        elif(urlparts.password is None and urlparts.username == "anonymous"):
            sftp_password = "anonymous"
        else:
            sftp_password = ""
        if(urlparts.scheme != "sftp"):
            return False
        ssh = paramiko.SSHClient()
        ssh.load_system_host_keys()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        try:
            ssh.connect(urlparts.hostname, port=sftp_port,
                        username=urlparts.username, password=urlparts.password)
        except paramiko.ssh_exception.SSHException:
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        sftp = ssh.open_sftp()
        sftp.putfo(sftpfile, urlparts.path)
        sftp.close()
        ssh.close()
        sftpfile.seek(0, 0)
        return sftpfile
else:
    def upload_file_to_sftp_file(sftpfile, url):
        return False

if(haveparamiko):
    def upload_file_to_sftp_string(sftpstring, url):
        sftpfileo = BytesIO(sftpstring)
        sftpfile = upload_file_to_sftp_files(ftpfileo, url)
        sftpfileo.close()
        return sftpfile
else:
    def upload_file_to_sftp_string(url):
        return False


if(havepysftp):
    def download_file_from_pysftp_file(url):
        urlparts = urlparse.urlparse(url)
        file_name = os.path.basename(urlparts.path)
        file_dir = os.path.dirname(urlparts.path)
        if(urlparts.scheme == "http" or urlparts.scheme == "https"):
            return False
        sftp_port = urlparts.port
        if(urlparts.port is None):
            sftp_port = 22
        else:
            sftp_port = urlparts.port
        if(urlparts.username is not None):
            sftp_username = urlparts.username
        else:
            sftp_username = "anonymous"
        if(urlparts.password is not None):
            sftp_password = urlparts.password
        elif(urlparts.password is None and urlparts.username == "anonymous"):
            sftp_password = "anonymous"
        else:
            sftp_password = ""
        if(urlparts.scheme != "sftp"):
            return False
        try:
            pysftp.Connection(urlparts.hostname, port=sftp_port,
                              username=urlparts.username, password=urlparts.password)
        except paramiko.ssh_exception.SSHException:
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        sftp = ssh.open_sftp()
        sftpfile = BytesIO()
        sftp.getfo(urlparts.path, sftpfile)
        sftp.close()
        ssh.close()
        sftpfile.seek(0, 0)
        return sftpfile
else:
    def download_file_from_pysftp_file(url):
        return False

if(havepysftp):
    def download_file_from_pysftp_string(url):
        sftpfile = download_file_from_pysftp_file(url)
        return sftpfile.read()
else:
    def download_file_from_ftp_string(url):
        return False

if(havepysftp):
    def upload_file_to_pysftp_file(sftpfile, url):
        urlparts = urlparse.urlparse(url)
        file_name = os.path.basename(urlparts.path)
        file_dir = os.path.dirname(urlparts.path)
        sftp_port = urlparts.port
        if(urlparts.scheme == "http" or urlparts.scheme == "https"):
            return False
        if(urlparts.port is None):
            sftp_port = 22
        else:
            sftp_port = urlparts.port
        if(urlparts.username is not None):
            sftp_username = urlparts.username
        else:
            sftp_username = "anonymous"
        if(urlparts.password is not None):
            sftp_password = urlparts.password
        elif(urlparts.password is None and urlparts.username == "anonymous"):
            sftp_password = "anonymous"
        else:
            sftp_password = ""
        if(urlparts.scheme != "sftp"):
            return False
        try:
            pysftp.Connection(urlparts.hostname, port=sftp_port,
                              username=urlparts.username, password=urlparts.password)
        except paramiko.ssh_exception.SSHException:
            return False
        except socket.gaierror:
            log.info("Error With URL "+httpurl)
            return False
        except socket.timeout:
            log.info("Error With URL "+httpurl)
            return False
        sftp = ssh.open_sftp()
        sftp.putfo(sftpfile, urlparts.path)
        sftp.close()
        ssh.close()
        sftpfile.seek(0, 0)
        return sftpfile
else:
    def upload_file_to_pysftp_file(sftpfile, url):
        return False

if(havepysftp):
    def upload_file_to_pysftp_string(sftpstring, url):
        sftpfileo = BytesIO(sftpstring)
        sftpfile = upload_file_to_pysftp_files(ftpfileo, url)
        sftpfileo.close()
        return sftpfile
else:
    def upload_file_to_pysftp_string(url):
        return False
