what.utils.file API documentation

# what.utils.file

View Source

     1import os.path
     2import hashlib
     3
     4import progressbar
     5import urllib.request
     6
     7class MyProgressBar():
     8    def __init__(self):
     9        self.pbar = None
    10
    11    def __call__(self, block_num, block_size, total_size):
    12        if not self.pbar:
    13            self.pbar=progressbar.ProgressBar(maxval=total_size)
    14            self.pbar.start()
    15
    16        downloaded = block_num * block_size
    17        if downloaded < total_size:
    18            self.pbar.update(downloaded)
    19        else:
    20            self.pbar.finish()
    21
    22def download_file(file_name, path, url, hash):
    23    print('Downloading', os.path.join(path, file_name))
    24
    25    try:
    26        try:
    27            urllib.request.urlretrieve(url, os.path.join(path, file_name), MyProgressBar())
    28        except Exception as e:
    29            print(e)
    30    except (Exception, KeyboardInterrupt):
    31        if os.path.exists(os.path.join(path, file_name)):
    32            os.remove(os.path.join(path, file_name))
    33        raise
    34
    35    # Validate download if succeeded and user provided an expected hash
    36    sha256_hash = hashlib.sha256()
    37    with open(os.path.join(path, file_name),"rb") as f:
    38        for byte_block in iter(lambda: f.read(4096),b""):
    39            sha256_hash.update(byte_block)
    40        if (sha256_hash.hexdigest() == hash):
    41            print("Model {} downloaded".format(os.path.join(path, file_name)))
    42        else:
    43            print('Incomplete or corrupted file detected.')
    44
    45def get_file(file_name, path, url, hash):
    46    if os.path.isfile(os.path.join(path, file_name)):
    47        sha256_hash = hashlib.sha256()
    48        with open(os.path.join(path, file_name),"rb") as f:
    49            for byte_block in iter(lambda: f.read(4096),b""):
    50                sha256_hash.update(byte_block)
    51            if (sha256_hash.hexdigest() == hash):
    52                print("Model {} already exists".format(os.path.join(path, file_name)))
    53            else:
    54                redownload = input("File corrupted, Redownload? [Y/n]")
    55                if len(redownload) == 0:
    56                    redownload = 'y'
    57                else:
    58                    while not redownload in ['y', 'n']:
    59                        redownload = input("File corrupted, Redownload? [y or n]")
    60
    61                if redownload == 'y':
    62                    download_file(file_name, path, url, hash)
    63
    64    else:
    65        download_file(file_name, path, url, hash)

class MyProgressBar: View Source

     8class MyProgressBar():
     9    def __init__(self):
    10        self.pbar = None
    11
    12    def __call__(self, block_num, block_size, total_size):
    13        if not self.pbar:
    14            self.pbar=progressbar.ProgressBar(maxval=total_size)
    15            self.pbar.start()
    16
    17        downloaded = block_num * block_size
    18        if downloaded < total_size:
    19            self.pbar.update(downloaded)
    20        else:
    21            self.pbar.finish()

def download_file(file_name, path, url, hash): View Source

    23def download_file(file_name, path, url, hash):
    24    print('Downloading', os.path.join(path, file_name))
    25
    26    try:
    27        try:
    28            urllib.request.urlretrieve(url, os.path.join(path, file_name), MyProgressBar())
    29        except Exception as e:
    30            print(e)
    31    except (Exception, KeyboardInterrupt):
    32        if os.path.exists(os.path.join(path, file_name)):
    33            os.remove(os.path.join(path, file_name))
    34        raise
    35
    36    # Validate download if succeeded and user provided an expected hash
    37    sha256_hash = hashlib.sha256()
    38    with open(os.path.join(path, file_name),"rb") as f:
    39        for byte_block in iter(lambda: f.read(4096),b""):
    40            sha256_hash.update(byte_block)
    41        if (sha256_hash.hexdigest() == hash):
    42            print("Model {} downloaded".format(os.path.join(path, file_name)))
    43        else:
    44            print('Incomplete or corrupted file detected.')

def get_file(file_name, path, url, hash): View Source

    46def get_file(file_name, path, url, hash):
    47    if os.path.isfile(os.path.join(path, file_name)):
    48        sha256_hash = hashlib.sha256()
    49        with open(os.path.join(path, file_name),"rb") as f:
    50            for byte_block in iter(lambda: f.read(4096),b""):
    51                sha256_hash.update(byte_block)
    52            if (sha256_hash.hexdigest() == hash):
    53                print("Model {} already exists".format(os.path.join(path, file_name)))
    54            else:
    55                redownload = input("File corrupted, Redownload? [Y/n]")
    56                if len(redownload) == 0:
    57                    redownload = 'y'
    58                else:
    59                    while not redownload in ['y', 'n']:
    60                        redownload = input("File corrupted, Redownload? [y or n]")
    61
    62                if redownload == 'y':
    63                    download_file(file_name, path, url, hash)
    64
    65    else:
    66        download_file(file_name, path, url, hash)