import io
import logging
import mimetypes
import os
import re
import threading
from glob import has_magic
from typing import Optional, Generator, List, Union, Dict, Collection, Any, Tuple, Callable

from fsspec import AbstractFileSystem, Callback
from fsspec.callbacks import NoOpCallback
from fsspec.spec import AbstractBufferedFile
from fsspec.utils import tokenize

import tosnativeclient
from .tos_client import TosClient

log = logging.getLogger(__name__)

_DEFAULT_CALLBACK = NoOpCallback()

_MIN_UPLOAD_PART_SIZE = 4 * 1024 * 1024
_DEFAULT_READ_CHUNK_SIZE = 1 * 1024 * 1024
_COMMON_IO_SIZE = _DEFAULT_READ_CHUNK_SIZE


class TosFileSystem(AbstractFileSystem):
    protocol = ('tos', 'tos-new')

    def __init__(self, region: str, endpoint: str = '', key='', secret='', part_size: int = 8 * 1024 * 1024,
                 max_retry_count: int = 3, shared_prefetch_tasks: int = 32, shared_upload_part_tasks: int = 32,
                 shared_upload_part_copy_tasks: int = 32, enable_crc: bool = True, security_token: Optional[str] = None,
                 **kwargs):
        super().__init__(**kwargs)
        if part_size > 0 and part_size < _MIN_UPLOAD_PART_SIZE:
            raise ValueError('The part_size must be >= {}'.format(_MIN_UPLOAD_PART_SIZE))

        self._client = TosClient(region, endpoint, key, secret, part_size, max_retry_count, shared_prefetch_tasks,
                                 shared_upload_part_tasks, shared_upload_part_copy_tasks, enable_crc,
                                 security_token=security_token)

    def _open(
            self,
            path: str,
            mode: str = 'rb',
            block_size: Optional[int] = None,
            autocommit: bool = True,
            cache_options: Optional[dict] = None,
            size: Optional[int] = None,
            etag: Optional[str] = None,
            crc64: Optional[int] = None,
            **kwargs,
    ) -> AbstractBufferedFile:
        log.debug(f'open {path} with mode {mode}')
        return TosFile(self, path, mode, block_size, autocommit, cache_options, size, etag, crc64, **kwargs)

    def put_file(
            self,
            lpath: str,
            rpath: str,
            callback: Optional[Callback] = _DEFAULT_CALLBACK,
            **kwargs,
    ) -> None:
        """
        upload local file to remote file on tos
        """
        log.debug(f'put_file from {lpath} to {rpath}')
        if not os.path.exists(lpath):
            raise FileNotFoundError(f'Local file {lpath} not found')

        if os.path.isdir(lpath):
            self.makedirs(rpath, exist_ok=True)
            return None

        size = os.path.getsize(lpath)
        callback.set_size(size)

        if 'ContentType' not in kwargs:
            content_type, _ = mimetypes.guess_type(lpath)
            if content_type is not None:
                kwargs['ContentType'] = content_type

        with open(lpath, 'rb') as f1:
            with self.open(rpath, 'wb', **kwargs) as f2:
                while 1:
                    chunk = f1.read(_COMMON_IO_SIZE)
                    if not chunk:
                        break
                    written = f2.write(chunk)
                    callback.relative_update(written)

        self.invalidate_cache(rpath)

    def get_file(self, rpath: str, lpath: str, callback: Optional[Callback] = _DEFAULT_CALLBACK, **kwargs) -> None:
        """
        download remote file on tos to local file
        """
        log.debug(f'get_file from {rpath} to {lpath}')
        if os.path.isdir(lpath):
            log.warning('local path is dir, skip to get file')
            return

        ldir = os.path.dirname(lpath)
        if ldir:
            os.makedirs(ldir, exist_ok=True)

        bucket, key = _parse_tos_url(rpath)
        if not key:
            raise ValueError('Attempt to lookup non key-like remote path')

        inode = self._client.lookup(bucket, key)
        if inode is None:
            raise FileNotFoundError(f'{rpath} not found')

        if not inode.is_file:
            log.warning('remote path is dir, skip to get file')
            return

        callback.set_size(inode.size)
        kwargs['size'] = inode.size
        kwargs['etag'] = inode.etag
        kwargs['crc64'] = inode.crc64
        # sequential read and do not need with_sliding_window
        kwargs['with_sliding_window'] = False
        with self.open(rpath, 'rb', **kwargs) as f1:
            with open(lpath, 'wb') as f2:
                while 1:
                    chunk = f1.read(_COMMON_IO_SIZE)
                    if not chunk:
                        break
                    written = f2.write(chunk)
                    callback.relative_update(written)

    def cp_file(self, path1: str, path2: str, reserve_metadata: bool = False, **kwargs) -> None:
        """
        copy remote file between locations on tos
        """
        log.debug(f'cp_file from {path1} to {path2}')
        src_bucket, src_key = _parse_tos_url(path1)
        dst_bucket, dst_key = _parse_tos_url(path2)
        if not src_key or not dst_key:
            raise ValueError('Attempt to copy non key-like remote path')

        recursive = kwargs.get('recursive', False)
        self._client.copy_object(dst_bucket, dst_key, src_bucket=src_bucket, src_key=src_key,
                                 copy_metadata=reserve_metadata, recursive=recursive)
        self.invalidate_cache(path2)

    def mkdir(self, path: str, create_parents: bool = True, create_bucket: bool = False, **kwargs) -> None:
        log.debug(f'mkdir for {path}')
        bucket, folder_key = _parse_tos_url(path)
        if not folder_key:
            if create_bucket:
                self._client.create_bucket(bucket, **kwargs)
                self.invalidate_cache(path)
                return

            raise ValueError('Attempt to create non key-like remote path')
        check_exists = kwargs.get('check_exists', False)
        self._client.mkdir(bucket, folder_key, create_parents=create_parents, check_exists=check_exists)
        self.invalidate_cache(path)

    def makedirs(self, path: str, exist_ok: bool = False) -> None:
        log.debug(f'makedirs for {path}')
        try:
            self.mkdir(path, create_parents=True, check_exists=True)
        except FileExistsError:
            if not exist_ok:
                raise

    def touch(self, path: str, truncate: bool = True, **kwargs) -> None:
        log.debug(f'touch for {path}')
        bucket, key = _parse_tos_url(path)
        if not key:
            raise ValueError('Attempt to create non key-like remote path')

        if not truncate and self.exists(path):
            raise ValueError('Currently does not support touching existent files')

        content_type = kwargs.get('content_type', None)
        if content_type is None:
            content_type, _ = mimetypes.guess_type(path)
        self._client.touch(bucket, key, content_type=content_type)
        self.invalidate_cache(path)

    def mv(self, path1: str, path2: str, recursive: bool = False, maxdepth: Optional[int] = None,
           reserve_metadata: bool = False, **kwargs) -> None:
        log.debug(f'mv from {path1} to {path2}')
        if path1 == path2:
            log.debug('the paths are the same, so no files were moved')
            return

        if maxdepth is not None:
            if maxdepth < 1:
                raise ValueError('The maxdepth must be at least 1')
            bucket, _ = _parse_tos_url(path1)
            bucket_type = self._client.get_bucket_type(bucket)
            if bucket_type == 'hns':
                raise ValueError('The maxdepth is not supported for hns bucket')

            self.copy(path1, path2, recursive=recursive, maxdepth=maxdepth, reserve_metadata=reserve_metadata, **kwargs)
            self.rm(path1, recursive=recursive, maxdepth=maxdepth, bucket_type=bucket_type)
            return

        src_bucket, src_key = _parse_tos_url(path1)
        dst_bucket, dst_key = _parse_tos_url(path2)
        if src_bucket == dst_bucket and src_key == dst_key:
            log.debug('the paths are the same, so no files were moved')
            return

        self._client.rename(dst_bucket, dst_key, src_bucket=src_bucket, src_key=src_key, recursive=recursive,
                            copy_metadata=reserve_metadata)
        self.invalidate_cache(path1)
        self.invalidate_cache(path2)

    def copy(self, path1: Union[str, List[str]], path2: str, recursive: bool = False, maxdepth: Optional[int] = None,
             on_error: Optional[Union[str, Callable[[Exception], None]]] = None, reserve_metadata: bool = False,
             **kwargs) -> None:
        if isinstance(path1, str):
            paths1 = [path1]
        elif isinstance(path1, list):
            paths1 = path1
        else:
            raise TypeError(f'path1 is not a str or a list')

        # handle multi sources to single target
        if len(paths1) > 1 and not path2.endswith('/'):
            path2 += '/'

        for path1 in paths1:
            if has_magic(path1):
                self._do_copy_with_magic(path1, path2, recursive=recursive, on_error=on_error,
                                         reserve_metadata=reserve_metadata, **kwargs)
            else:
                self._do_copy(path1, path2, recursive, maxdepth, on_error, reserve_metadata, **kwargs)

    def _do_copy_with_magic(self, path1: str, path2: str, recursive: bool = False,
                            on_error: Optional[Union[str, Callable[[Exception], None]]] = None,
                            reserve_metadata: bool = False, **kwargs):
        from fsspec.utils import other_paths

        if on_error is None and recursive:
            on_error = 'ignore'
        elif on_error is None:
            on_error = 'raise'

        source_is_str = isinstance(path1, str)
        paths = self.expand_path(path1, recursive=recursive)
        if source_is_str and not recursive:
            # Non-recursive glob does not copy directories
            paths = [p for p in paths if not (_trailing_sep(p) or self.isdir(p))]
            if not paths:
                return

        isdir = isinstance(path2, str) and (_trailing_sep(path2) or self.isdir(path2))
        path2 = other_paths(
            paths,
            path2,
            exists=isdir and source_is_str and not _trailing_sep_maybe_asterisk(path1),
            is_dir=isdir,
            flatten=not source_is_str,
        )
        for p1, p2 in zip(paths, path2):
            try:
                self.cp_file(p1, p2, reserve_metadata=reserve_metadata, recursive=recursive, **kwargs)
            except FileNotFoundError:
                if on_error == 'raise':
                    raise

    def _do_copy(self, path1: Union[str, List[str]], path2: str, recursive: bool = False,
                 maxdepth: Optional[int] = None,
                 on_error: Optional[Union[str, Callable[[Exception], None]]] = None, reserve_metadata: bool = False,
                 **kwargs) -> None:
        log.debug(f'copy from {path1} to {path2}')
        if path1 == path2:
            log.debug('the paths are the same, so no files were copied')
            return

        if maxdepth is not None and maxdepth < 1:
            raise ValueError('The maxdepth must be at least 1')
        src_bucket, src_key = _parse_tos_url(path1)
        dst_bucket, dst_key = _parse_tos_url(path2)
        if src_bucket == dst_bucket and src_key == dst_key:
            log.debug('the paths are the same, so no files were copied')
            return

        try:
            self._client.copy(dst_bucket, dst_key, src_bucket=src_bucket, src_key=src_key, recursive=recursive,
                              copy_metadata=reserve_metadata, maxdepth=maxdepth, with_src_parent=False)
        except Exception as ex:
            if on_error is None or on_error == 'raise':
                raise
            if callable(on_error):
                on_error(ex)
        self.invalidate_cache(path2)

    def rm(self, path: Union[str, List[str]], recursive: bool = False, maxdepth: Optional[int] = None,
           bucket_type: Optional[str] = None) -> None:
        if isinstance(path, str):
            paths = [path]
        elif isinstance(path, list):
            paths = path
        else:
            raise TypeError(f'path is not a str or a list')

        for path in paths:
            self._do_rm(path, recursive, maxdepth, bucket_type)

    def _do_rm(self, path: str, recursive: bool = False, maxdepth: Optional[int] = None,
               bucket_type: Optional[str] = None) -> None:
        log.debug(f'rm for {path}')
        if maxdepth is not None:
            if maxdepth < 1:
                raise ValueError('The maxdepth must be at least 1')
            bucket, _ = _parse_tos_url(path)
            if bucket_type is None:
                bucket_type = self._client.get_bucket_type(bucket)

            if bucket_type == 'hns':
                raise ValueError('The maxdepth is not supported for hns bucket')

            for (prefix, dirs, files) in self.walk(path, maxdepth=maxdepth, sort_names=False, use_full_path=True,
                                                   detail=False):
                for file in files:
                    self.rm_file(file)

                for d in dirs:
                    self.rm_file(d)

                self.rm_file(prefix)
            return

        bucket, folder_key = _parse_tos_url(path)
        self._client.rm(bucket, folder_key, recursive=recursive)
        self.invalidate_cache(path)

    def rmdir(self, path: str, **kwargs) -> None:
        """
        Remove a directory, if empty
        """
        log.debug(f'rmdir for {path}')
        bucket, folder_key = _parse_tos_url(path)
        if not folder_key:
            if kwargs.get('delete_bucket', False):
                self._client.delete_bucket(bucket)
                self.invalidate_cache(path)
                return

            raise ValueError('Attempt to remove non key-like remote path')
        self._client.delete_folder(bucket, folder_key)
        self.invalidate_cache(path)

    def _rm(self, path: str) -> None:
        """
        Delete one file
        """
        log.debug(f'_rm for {path}')
        bucket, key = _parse_tos_url(path)
        if not key:
            raise ValueError('Attempt to remove non key-like remote path')
        self._client.delete_object(bucket, key)
        self.invalidate_cache(path)

    def ls(self, path: str, detail: bool = True, sort_names: bool = True, **kwargs) -> Union[List[Dict], List[str]]:
        log.debug(f'ls for {path}')
        if path in ['/', '']:
            buckets = [_trans_tos_bucket(tos_bucket) for tos_bucket in self._client.list_buckets()]
            return buckets if detail else sorted([bucket['name'] for bucket in buckets])

        objects = []
        bucket, prefix = _parse_tos_url(path)
        if prefix and not prefix.endswith('/'):
            prefix += '/'
        # exclude self
        list_stream = self._client.list_objects(bucket, prefix, delimiter='/', start_after=prefix)
        dup_keys = set()
        try:
            while 1:
                result = next(list_stream)
                if not result:
                    break

                for common_prefix in result[0].common_prefixes:
                    objects.append(_trans_common_prefix(bucket, common_prefix))
                    dup_keys.add(common_prefix)

                for tos_object in result[0].contents:
                    if tos_object.key in dup_keys:
                        continue
                    objects.append(_trans_tos_object(tos_object))
        except StopIteration:
            pass
        finally:
            list_stream.close()
        if detail:
            return objects

        names = [obj['name'] for obj in objects]
        if sort_names:
            return sorted(names)

        return names

    def walk(self, path: str, maxdepth: Optional[int] = None, detail: bool = False, topdown: bool = True,
             on_error: Optional[Union[str, Callable[[Exception], None]]] = None, sort_names: bool = True,
             use_full_path: bool = False, **kwargs) -> \
            Generator[str, Union[List[Dict], List[str]], Union[List[Dict], List[str]]]:
        log.debug(f'walk for {path}')
        if maxdepth is not None and maxdepth < 1:
            raise ValueError('The maxdepth must be at least 1')

        if not topdown:
            raise ValueError('The topdown must be True')

        bucket, prefix = _parse_tos_url(path)
        if prefix and not prefix.endswith('/'):
            prefix += '/'

        walk_stream = self._client.walk(bucket, prefix, maxdepth=maxdepth)
        try:
            while 1:
                result = next(walk_stream)
                if not result:
                    break
                dirs = [_trans_common_prefix(bucket, common_prefix, use_full_path) for common_prefix in
                        result[1].common_prefixes] if detail else sorted(
                    [_normalize_path('/'.join([bucket, common_prefix]), use_full_path) for common_prefix in
                     result[1].common_prefixes]) if sort_names else [
                    _normalize_path('/'.join([bucket, common_prefix]), use_full_path) for common_prefix
                    in result[1].common_prefixes]
                files = [_trans_tos_object_directly(tos_object, use_full_path) for tos_object in
                         result[1].contents] if detail else sorted(
                    [_normalize_path('/'.join([tos_object.bucket, tos_object.key]), use_full_path) for tos_object in
                     result[1].contents]) if sort_names else [
                    _normalize_path('/'.join([tos_object.bucket, tos_object.key]), use_full_path) for
                    tos_object in result[1].contents]
                yield _normalize_path('/'.join([bucket, result[0]])), dirs, files
        except StopIteration:
            pass
        except Exception as ex:
            if on_error is None or on_error == 'raise':
                raise
            if callable(on_error):
                on_error(ex)
        finally:
            walk_stream.close()

    def find(self, path: str, maxdepth: Optional[int] = None, withdirs: bool = False, detail: bool = False,
             sort_names: bool = True, **kwargs) -> \
            Union[List[str], Dict]:
        log.debug(f'find for {path}')
        bucket, prefix = _parse_tos_url(path)
        if not prefix:
            raise ValueError('Cannot traverse all of TOS')
        elif not prefix.endswith('/'):
            prefix += '/'

        if maxdepth is not None or withdirs:
            return self._find_by_walk(path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, sort_names=sort_names,
                                      **kwargs)

        bucket_type = self._client.get_bucket_type(bucket)
        if bucket_type == 'hns':
            return self._find_by_walk(path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, sort_names=sort_names,
                                      **kwargs)

        objects = {}
        names = []
        # exclude self
        list_stream = self._client.list_objects(bucket, prefix, start_after=prefix)
        try:
            while 1:
                result = next(list_stream)
                if not result:
                    break

                if detail:
                    for tos_object in result[0].contents:
                        if tos_object.key.endswith('/'):
                            continue
                        obj = _trans_tos_object_directly(tos_object)
                        objects[obj['name']] = obj
                else:
                    for tos_object in result[0].contents:
                        names.append(_normalize_path('/'.join([tos_object.bucket, tos_object.key])))
        except StopIteration:
            pass
        finally:
            list_stream.close()

        if detail:
            return objects

        if sort_names:
            return sorted(names)

        return names

    def _find_by_walk(self, path: str, maxdepth: Optional[int] = None, withdirs: bool = False,
                      detail: bool = False, sort_names: bool = True, **kwargs) -> \
            Union[List[str], Dict]:
        if detail:
            objects = {}
            for (prefix, dirs, files) in self.walk(path, maxdepth=maxdepth, detail=detail, sort_names=False,
                                                   use_full_path=True):
                for file in files:
                    objects[file['name']] = file

                if withdirs:
                    for d in dirs:
                        objects[d['name']] = d

            return objects

        names = []
        for (prefix, dirs, files) in self.walk(path, maxdepth=maxdepth, detail=detail, sort_names=False,
                                               use_full_path=True):
            for file in files:
                names.append(file)

            if withdirs:
                for d in dirs:
                    names.append(d)

        if sort_names:
            return sorted(names)
        return names

    def glob(self, path: str, maxdepth: Optional[int] = None, **kwargs) -> Collection[Any]:
        log.debug(f'glob for {path}')
        bucket, prefix = _parse_tos_url(path)
        if not prefix:
            raise ValueError('Cannot traverse all of TOS')

        if maxdepth is not None and maxdepth < 1:
            raise ValueError('The maxdepth must be at least 1')

        seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,)
        ends_with_sep = path.endswith(seps)  # _strip_protocol strips trailing slash
        path = self._strip_protocol(path)
        append_slash_to_dirname = ends_with_sep or path.endswith(
            tuple(sep + '**' for sep in seps)
        )
        idx_star = path.find('*') if path.find('*') >= 0 else len(path)
        idx_qmark = path.find('?') if path.find('?') >= 0 else len(path)
        idx_brace = path.find('[') if path.find('[') >= 0 else len(path)

        min_idx = min(idx_star, idx_qmark, idx_brace)

        detail = kwargs.pop('detail', False)
        withdirs = kwargs.pop('withdirs', True)

        if not has_magic(path):
            if self.exists(path, **kwargs):
                return {path: self.info(path, **kwargs)} if detail else [path]
            return {} if detail else []

        if '/' in path[:min_idx]:
            min_idx = path[:min_idx].rindex('/')
            root = path[: min_idx + 1]
            depth = path[min_idx + 1:].count('/') + 1
        else:
            root = ''
            depth = path[min_idx + 1:].count('/') + 1

        if '**' in path:
            if maxdepth is not None:
                idx_double_stars = path.find('**')
                depth_double_stars = path[idx_double_stars:].count('/') + 1
                depth = depth - depth_double_stars + maxdepth
            else:
                depth = None

        allpaths = self.find(root, maxdepth=depth, withdirs=withdirs, detail=True, **kwargs)
        pattern_str = _glob_translate(path + ('/' if ends_with_sep else ''))
        pattern = re.compile(pattern_str)
        out = {
            p: info
            for p, info in sorted(allpaths.items())
            if pattern.match(
                p + '/'
                if append_slash_to_dirname and info['type'] == 'directory'
                else p
            )
        }
        return out if detail else list(out)

    def info(self, path: str, **kwargs) -> Dict:
        log.debug(f'info for {path}')
        if path in ['/', '']:
            return {'name': path, 'size': 0, 'type': 'directory'}
        bucket, key = _parse_tos_url(path)

        if not key:
            try:
                tos_bucket = self._client.head_bucket(bucket)
                return _trans_tos_bucket(tos_bucket)
            except tosnativeclient.TosException as ex:
                if len(ex.args) > 0 and ex.args[0].status_code == 404 and ex.args[0].ec == '0006-00000001':
                    raise FileNotFoundError(f'{path} not found')
                else:
                    raise ex

        inode = self._client.lookup(bucket, key)
        if inode is None:
            raise FileNotFoundError(f'{path} not found')
        return _trans_tos_inode(inode)

    def exists(self, path: str, **kwargs) -> bool:
        log.debug(f'exists for {path}')
        if path in ['/', '']:
            return True
        try:
            self.info(path)
            return True
        except FileNotFoundError:
            return False

    def checksum(self, path: str) -> int:
        log.debug(f'checksum for {path}')
        info = self.info(path)
        if info['type'] != 'directory':
            return int(info['etag'].strip('"').split('-')[0], 16)

        return int(tokenize(self.info(path)), 16)

    def isdir(self, path: str) -> bool:
        bucket, key = _parse_tos_url(path)
        if not key:
            return False
        inode = self._client.lookup(bucket, key)
        return False if inode is None else not inode.is_file

    def isfile(self, path: str) -> bool:
        bucket, key = _parse_tos_url(path)
        if not key:
            return False
        inode = self._client.lookup(bucket, key)
        return False if inode is None else inode.is_file

    def invalidate_cache(self, path: Optional[str] = None):
        if path is None:
            self.dircache.clear()
        else:
            path = self._strip_protocol(path)
            self.dircache.pop(path, None)
            while path:
                self.dircache.pop(path, None)
                path = self._parent(path)

    def close(self) -> None:
        self._client.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
        if exc_type is not None:
            try:
                log.info(f'exception occurred before closing tos client: {exc_type.__name__}: {exc_val}')
            except Exception:
                pass
            finally:
                self.close()
        else:
            self.close()


class TosFile(AbstractBufferedFile):
    def __init__(self, fs: TosFileSystem, path: str,
                 mode: str = 'rb',
                 block_size: Union[int, str] = 'default',
                 autocommit: bool = True,
                 cache_options: Optional[dict] = None,
                 size: Optional[int] = None,
                 etag: Optional[str] = None,
                 crc64: Optional[int] = None,
                 **kwargs):
        kwargs['cache_type'] = 'none'
        bucket, key = _parse_tos_url(path)
        if not key:
            raise ValueError('Attempt to open non key-like path: {}'.format(path))

        if mode not in {'ab', 'rb', 'wb'}:
            raise NotImplementedError('File mode not supported')

        if mode == 'rb':
            if size is None or etag is None or size == -1:
                tos_object = fs._client.head_object(bucket, key)
                etag = tos_object.etag
                size = tos_object.size
                crc64 = tos_object.crc64

        super().__init__(fs, path, mode, block_size=block_size, autocommit=autocommit, cache_options=cache_options,
                         size=size,
                         **kwargs)
        if 'r' not in self.mode and self.blocksize > 0 and self.blocksize < _MIN_UPLOAD_PART_SIZE:
            raise ValueError('The block_size must be >= {} in write mode'.format(_MIN_UPLOAD_PART_SIZE))

        self._bucket = bucket
        self._key = key
        self._etag = etag
        self._size = size
        self._crc64 = crc64
        self._lock = threading.Lock()
        self._use_append = False
        self._write_stream = None
        self._read_stream = None

        if 'r' in self.mode:
            with_sliding_window = kwargs.get('with_sliding_window', True)
            self._read_stream = self.fs._client.get_object(self._bucket, self._key, etag=self._etag, size=self._size,
                                                           crc64=self._crc64,
                                                           read_part_size=self.blocksize if self.blocksize > 0 else None,
                                                           with_sliding_window=with_sliding_window)
        else:
            if 'a' in self.mode:
                if self._size is None or self._etag is None or self._crc64 is None:
                    try:
                        tos_object = self.fs._client.head_object(bucket, key)
                        self._size = tos_object.size
                        self._etag = tos_object.etag
                        self._crc64 = tos_object.crc64
                        self._use_append = True
                    except tosnativeclient.TosException as ex:
                        if len(ex.args) > 0 and ex.args[0].status_code == 404 and ex.args[0].ec == '0017-00000003':
                            pass
                        else:
                            raise ex

            content_type = kwargs.get('ContentType', None)
            if self._use_append:
                self._write_stream = self.fs._client.append_object(self._bucket, self._key, self._etag, self._size,
                                                                   self._crc64,
                                                                   write_part_size=self.blocksize if self.blocksize > 0 else None,
                                                                   content_type=content_type)
            else:
                self._write_stream = self.fs._client.put_object(self._bucket, self._key,
                                                                write_part_size=self.blocksize if self.blocksize > 0 else None,
                                                                content_type=content_type)

    # ===== method for write mode =====
    def write(self, data: Union[bytes, memoryview, str]) -> int:
        self._check_write_status()
        if isinstance(data, memoryview):
            data = data.tobytes()
        elif isinstance(data, str):
            data = data.encode('utf-8')
        elif not isinstance(data, bytes):
            data = memoryview(data).tobytes()
        written = self._write_stream.write(data)
        assert written == len(data)
        self.loc += written
        return written

    def flush(self, force: bool = False) -> None:
        if self._write_stream:
            self._check_write_status()
            self._write_stream.flush(force)

    def commit(self) -> None:
        self.flush(self.autocommit)

    def discard(self) -> None:
        self._check_write_status()
        self._write_stream.abort()

    def _check_write_status(self) -> None:
        if not self.writable() or self._write_stream is None:
            raise ValueError('File is not writable')
        if self.closed:
            raise ValueError('I/O operation on closed file')

    # ===== method for read mode =====
    def seek(self, loc: int, whence: int = 0) -> int:
        self._check_seek_status()
        if whence == io.SEEK_END:
            if loc >= 0:
                self.loc = self._size
                return self.loc
            # offset is negative
            loc += self._size
        elif whence == io.SEEK_CUR:
            if loc >= 0 and self._is_read_to_end():
                return self.loc
            loc += self.loc
        elif whence == io.SEEK_SET:
            pass
        else:
            raise ValueError('Invalid whence, must be passed SEEK_CUR, SEEK_SET, or SEEK_END')

        if loc < 0:
            raise ValueError(f'invalid seek offset {loc}')
        self.loc = loc
        return self.loc

    def read(self, size: Optional[int] = None) -> bytes:
        self._check_read_status()
        if size is not None and not isinstance(size, int):
            raise TypeError(f'Argument should be integer or None, not {type(size)!r}')

        if self._is_read_to_end():
            return b''
        read_start = self.loc
        if size is None or size < 0:
            # means read all
            read_end = self._size
        else:
            read_end = min(read_start + size, self._size)

        buffer = io.BytesIO()

        def callback(data: bytes) -> bool:
            nonlocal buffer
            buffer.write(data)
            return True

        self._random_read(read_start, read_end, _DEFAULT_READ_CHUNK_SIZE, callback)
        self.loc += buffer.tell()
        return buffer.getvalue()

    def readinto(self, buf) -> int:
        self._check_read_status()
        size = len(buf)
        if self._is_read_to_end() or size == 0:
            return 0

        try:
            view = memoryview(buf)
            if view.readonly:
                raise TypeError(f'Argument must be a writable bytes-like object, not {type(buf).__name__}')
        except TypeError:
            raise TypeError(f'Argument must be a writable bytes-like object, not {type(buf).__name__}')

        read_start = self.loc
        read_end = min(read_start + len(buf), self._size)
        if read_start >= read_end:
            return 0

        readed = 0

        def callback(data: bytes) -> bool:
            nonlocal readed
            nonlocal view
            view[readed: readed + len(data)] = data
            readed += len(data)
            return True

        self._random_read(read_start, read_end, _DEFAULT_READ_CHUNK_SIZE, callback)
        self.loc += readed
        return readed

    def readuntil(self, char: bytes = b'\n', blocks: Optional[int] = None) -> bytes:
        self._check_read_status()
        if self._is_read_to_end():
            return b''
        read_start = self.loc
        read_end = self._size

        readed = 0
        out = []

        def callback(data: bytes) -> bool:
            nonlocal readed
            nonlocal out
            found = data.find(char)
            if found > -1:
                out.append(data[: found + len(char)])
                self.seek(read_start + found + len(char))
                readed = 0
                return False

            readed += len(data)
            out.append(data)
            return True

        self._random_read(read_start, read_end, blocks or _DEFAULT_READ_CHUNK_SIZE, callback)
        self.loc += readed
        return b''.join(out)

    def _random_read(self, read_start, read_end, chunk_size, callback: Callable[[bytes], bool]) -> None:
        if chunk_size <= 0:
            chunk_size = _DEFAULT_READ_CHUNK_SIZE
        offset = read_start
        while 1:
            if offset >= read_end:
                break
            length = chunk_size if offset + chunk_size <= read_end else read_end - offset
            chunk = self._read_stream.read(offset, length)
            if not chunk:
                break
            if not callback(chunk):
                break
            offset += len(chunk)

    def _is_read_to_end(self) -> bool:
        return self.loc == self._size

    def _check_seek_status(self) -> None:
        if not self.seekable() or self._read_stream is None:
            raise ValueError('File is not seekable')
        if self.closed:
            raise ValueError('I/O operation on closed file')

    def _check_read_status(self) -> None:
        if not self.readable() or self._read_stream is None:
            raise ValueError('File is not readable')
        if self.closed:
            raise ValueError('I/O operation on closed file')

    # ===== common method for read/write mode =====
    def close(self):
        if self.closed:
            return
        with self._lock:
            if not self.closed:
                try:
                    if self.mode == 'rb':
                        self.cache = None
                        if self._read_stream:
                            self._read_stream.close()
                    else:
                        if self._write_stream:
                            self._write_stream.close()

                        if self.fs is not None:
                            self.fs.invalidate_cache(self.path)
                finally:
                    self.closed = True

    @property
    def bucket(self) -> str:
        return self._bucket

    @property
    def key(self) -> str:
        return self._key

    def _upload_chunk(self, final: bool = False) -> bool:
        raise NotImplementedError

    def _initiate_upload(self) -> None:
        raise NotImplementedError

    def _fetch_range(self, start: int, end: int) -> bytes:
        raise NotImplementedError


def _trans_tos_bucket(tos_bucket: tosnativeclient.TosBucket) -> Dict:
    return {
        'name': tos_bucket.name,
        'size': 0,
        'type': 'directory',
        'project_name': tos_bucket.project_name,
        'bucket_type': tos_bucket.bucket_type,
    }


def _normalize_path(full_path: str, use_full_path: bool = True) -> str:
    return full_path.rstrip('/') if use_full_path else full_path.rstrip('/').rsplit('/', 1)[-1]


def _trans_tos_inode(tos_inode: tosnativeclient.TosInode, use_full_path: bool = True) -> Dict:
    name = '/'.join([tos_inode.bucket, tos_inode.key])
    if not tos_inode.is_file:
        return {
            'name': _normalize_path(name, use_full_path),
            'key': tos_inode.key,
            'size': 0,
            'type': 'directory',
            'etag': 'd41d8cd98f00b204e9800998ecf8427e',
            'crc64': 0,
            'last_modified': tos_inode.last_modified,
        }

    return {
        'name': _normalize_path(name, use_full_path),
        'key': tos_inode.key,
        'size': tos_inode.size,
        'type': 'file',
        'etag': tos_inode.etag,
        'crc64': tos_inode.crc64,
        'last_modified': tos_inode.last_modified,
    }


def _trans_common_prefix(bucket: str, common_prefix: str, use_full_path: bool = True) -> Dict:
    name = '/'.join([bucket, common_prefix])
    return {
        'name': _normalize_path(name, use_full_path),
        'key': common_prefix,
        'size': 0,
        'type': 'directory',
        'etag': 'd41d8cd98f00b204e9800998ecf8427e',
        'crc64': 0,
    }


def _trans_tos_object(tos_object: tosnativeclient.TosObject, use_full_path: bool = True) -> Dict:
    if tos_object.key.endswith('/'):
        name = '/'.join([tos_object.bucket, tos_object.key])
        return {
            'name': _normalize_path(name, use_full_path),
            'key': tos_object.key,
            'size': 0,
            'type': 'directory',
            'etag': 'd41d8cd98f00b204e9800998ecf8427e',
            'crc64': 0,
            'last_modified': tos_object.last_modified,
            'object_type': tos_object.object_type,
            'storage_class': tos_object.storage_class,
        }

    return _trans_tos_object_directly(tos_object, use_full_path)


def _trans_tos_object_directly(tos_object: tosnativeclient.TosObject, use_full_path: bool = True) -> Dict:
    name = '/'.join([tos_object.bucket, tos_object.key])
    return {
        'name': _normalize_path(name, use_full_path),
        'key': tos_object.key,
        'size': tos_object.size,
        'type': 'file',
        'etag': tos_object.etag,
        'crc64': tos_object.crc64,
        'last_modified': tos_object.last_modified,
        'object_type': tos_object.object_type,
        'storage_class': tos_object.storage_class,
    }


def _parse_tos_url(url: str) -> Tuple[str, str]:
    if not url:
        raise ValueError('url is empty')

    if url.startswith('tos://'):
        url = url[len('tos://'):]

    if not url:
        raise ValueError('bucket is empty')

    url = url.split('/', maxsplit=1)
    if len(url) == 1:
        bucket = url[0]
        prefix = ''
    else:
        bucket = url[0]
        prefix = url[1]

    if not bucket:
        raise ValueError('bucket is empty')
    return bucket, prefix


def _translate(pat, STAR, QUESTION_MARK):
    # Copied from: https://github.com/python/cpython/pull/106703.
    res: list[str] = []
    add = res.append
    i, n = 0, len(pat)
    while i < n:
        c = pat[i]
        i = i + 1
        if c == '*':
            # compress consecutive `*` into one
            if (not res) or res[-1] is not STAR:
                add(STAR)
        elif c == '?':
            add(QUESTION_MARK)
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j + 1
            if j < n and pat[j] == ']':
                j = j + 1
            while j < n and pat[j] != ']':
                j = j + 1
            if j >= n:
                add('\\[')
            else:
                stuff = pat[i:j]
                if '-' not in stuff:
                    stuff = stuff.replace('\\', r'\\')
                else:
                    chunks = []
                    k = i + 2 if pat[i] == '!' else i + 1
                    while True:
                        k = pat.find('-', k, j)
                        if k < 0:
                            break
                        chunks.append(pat[i:k])
                        i = k + 1
                        k = k + 3
                    chunk = pat[i:j]
                    if chunk:
                        chunks.append(chunk)
                    else:
                        chunks[-1] += '-'
                    # Remove empty ranges -- invalid in RE.
                    for k in range(len(chunks) - 1, 0, -1):
                        if chunks[k - 1][-1] > chunks[k][0]:
                            chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:]
                            del chunks[k]
                    # Escape backslashes and hyphens for set difference (--).
                    # Hyphens that create ranges shouldn't be escaped.
                    stuff = '-'.join(
                        s.replace('\\', r'\\').replace('-', r'\-') for s in chunks
                    )
                # Escape set operations (&&, ~~ and ||).
                stuff = re.sub(r'([&~|])', r'\\\1', stuff)
                i = j + 1
                if not stuff:
                    # Empty range: never match.
                    add('(?!)')
                elif stuff == '!':
                    # Negated empty range: match any character.
                    add('.')
                else:
                    if stuff[0] == '!':
                        stuff = "^" + stuff[1:]
                    elif stuff[0] in ('^', '['):
                        stuff = '\\' + stuff
                    add(f'[{stuff}]')
        else:
            add(re.escape(c))
    assert i == n
    return res


def _glob_translate(pat: str):
    # Copied from: https://github.com/python/cpython/pull/106703.
    # The keyword parameters' values are fixed to:
    # recursive=True, include_hidden=True, seps=None
    """Translate a pathname with shell wildcards to a regular expression."""
    if os.path.altsep:
        seps = os.path.sep + os.path.altsep
    else:
        seps = os.path.sep
    escaped_seps = ''.join(map(re.escape, seps))
    any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
    not_sep = f'[^{escaped_seps}]'
    one_last_segment = f'{not_sep}+'
    one_segment = f'{one_last_segment}{any_sep}'
    any_segments = f'(?:.+{any_sep})?'
    any_last_segments = '.*'
    results = []
    parts = re.split(any_sep, pat)
    last_part_idx = len(parts) - 1
    for idx, part in enumerate(parts):
        if part == '*':
            results.append(one_segment if idx < last_part_idx else one_last_segment)
            continue
        if part == '**':
            results.append(any_segments if idx < last_part_idx else any_last_segments)
            continue
        elif '**' in part:
            raise ValueError(
                'Invalid pattern: \'**\' can only be an entire path component'
            )
        if part:
            results.extend(_translate(part, f'{not_sep}*', not_sep))
        if idx < last_part_idx:
            results.append(any_sep)
    res = ''.join(results)
    return rf'(?s:{res})\Z'


def _trailing_sep(path: str):
    return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep))


def _trailing_sep_maybe_asterisk(path: str):
    return path.endswith((os.sep, os.sep + '*')) or (
            os.altsep is not None and path.endswith((os.altsep, os.altsep + '*'))
    )
