what.utils.file API documentation

# what.utils.file

View Source

     1import os.path
     2import hashlib
     3
     4import progressbar
     5import urllib.request
     6
     7class __MyProgressBar__():
     8    def __init__(self):
     9        self.pbar = None
    10
    11    def __call__(self, block_num, block_size, total_size):
    12        if not self.pbar:
    13            self.pbar=progressbar.ProgressBar(maxval=total_size)
    14            self.pbar.start()
    15
    16        downloaded = block_num * block_size
    17        if downloaded < total_size:
    18            self.pbar.update(downloaded)
    19        else:
    20            self.pbar.finish()
    21
    22def download_file(file_name, path, url, hash):
    23    print('Downloading', os.path.join(path, file_name))
    24
    25    if not os.path.exists(path):
    26        os.makedirs(path)
    27
    28    try:
    29        try:
    30            urllib.request.urlretrieve(url, os.path.join(path, file_name), __MyProgressBar__())
    31        except Exception as e:
    32            print(e)
    33    except (Exception, KeyboardInterrupt):
    34        if os.path.exists(os.path.join(path, file_name)):
    35            os.remove(os.path.join(path, file_name))
    36        raise
    37
    38    # Validate download if succeeded and user provided an expected hash
    39    sha256_hash = hashlib.sha256()
    40    with open(os.path.join(path, file_name),"rb") as f:
    41        for byte_block in iter(lambda: f.read(4096),b""):
    42            sha256_hash.update(byte_block)
    43        if (sha256_hash.hexdigest() == hash):
    44            print("Model {} downloaded".format(os.path.join(path, file_name)))
    45        else:
    46            print('Incomplete or corrupted file detected.')
    47
    48def get_file(file_name, path, url, hash):
    49    if os.path.isfile(os.path.join(path, file_name)):
    50        sha256_hash = hashlib.sha256()
    51        with open(os.path.join(path, file_name),"rb") as f:
    52            for byte_block in iter(lambda: f.read(4096),b""):
    53                sha256_hash.update(byte_block)
    54            if (sha256_hash.hexdigest() == hash):
    55                print("Model {} already exists".format(os.path.join(path, file_name)))
    56            else:
    57                redownload = input("File corrupted, Redownload? [Y/n]")
    58                if len(redownload) == 0:
    59                    redownload = 'y'
    60                else:
    61                    while not redownload in ['y', 'n']:
    62                        redownload = input("File corrupted, Redownload? [y or n]")
    63
    64                if redownload == 'y':
    65                    download_file(file_name, path, url, hash)
    66
    67    else:
    68        download_file(file_name, path, url, hash)

def download_file(file_name, path, url, hash): View Source

    23def download_file(file_name, path, url, hash):
    24    print('Downloading', os.path.join(path, file_name))
    25
    26    if not os.path.exists(path):
    27        os.makedirs(path)
    28
    29    try:
    30        try:
    31            urllib.request.urlretrieve(url, os.path.join(path, file_name), __MyProgressBar__())
    32        except Exception as e:
    33            print(e)
    34    except (Exception, KeyboardInterrupt):
    35        if os.path.exists(os.path.join(path, file_name)):
    36            os.remove(os.path.join(path, file_name))
    37        raise
    38
    39    # Validate download if succeeded and user provided an expected hash
    40    sha256_hash = hashlib.sha256()
    41    with open(os.path.join(path, file_name),"rb") as f:
    42        for byte_block in iter(lambda: f.read(4096),b""):
    43            sha256_hash.update(byte_block)
    44        if (sha256_hash.hexdigest() == hash):
    45            print("Model {} downloaded".format(os.path.join(path, file_name)))
    46        else:
    47            print('Incomplete or corrupted file detected.')

def get_file(file_name, path, url, hash): View Source

    49def get_file(file_name, path, url, hash):
    50    if os.path.isfile(os.path.join(path, file_name)):
    51        sha256_hash = hashlib.sha256()
    52        with open(os.path.join(path, file_name),"rb") as f:
    53            for byte_block in iter(lambda: f.read(4096),b""):
    54                sha256_hash.update(byte_block)
    55            if (sha256_hash.hexdigest() == hash):
    56                print("Model {} already exists".format(os.path.join(path, file_name)))
    57            else:
    58                redownload = input("File corrupted, Redownload? [Y/n]")
    59                if len(redownload) == 0:
    60                    redownload = 'y'
    61                else:
    62                    while not redownload in ['y', 'n']:
    63                        redownload = input("File corrupted, Redownload? [y or n]")
    64
    65                if redownload == 'y':
    66                    download_file(file_name, path, url, hash)
    67
    68    else:
    69        download_file(file_name, path, url, hash)