# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['iotoolz', 'iotoolz.extensions']

package_data = \
{'': ['*']}

install_requires = \
['cachetools>=4.1.1,<5.0.0',
 'chardet>=3.0.4,<4.0.0',
 'cytoolz>=0.11.0,<0.12.0',
 'pydantic>=0.30',
 'python-dateutil>=2.8.1,<3.0.0',
 'requests-toolbelt>=0.9.1,<0.10.0',
 'requests>=2.24.0,<3.0.0',
 'toml>=0.10.1,<0.11.0',
 'typing-extensions']

extras_require = \
{':sys_platform == "darwin" or sys_platform == "windows"': ['python-magic-bin>=0.4.0,<0.5.0'],
 ':sys_platform == "linux"': ['python-magic>=0.4.0,<0.5.0'],
 'all': ['boto3>=1.16.5,<2.0.0', 'minio>=6.0.0,<7.0.0'],
 'boto3': ['boto3>=1.16.5,<2.0.0'],
 'minio': ['minio>=6.0.0,<7.0.0']}

setup_kwargs = {
    'name': 'iotoolz',
    'version': '0.1.0rc11',
    'description': 'Consistent io iterface to read and write from/to both local and different remote resources (e.g. http, s3)',
    'long_description': '# iotoolz\n\n[![PyPI version](https://badge.fury.io/py/iotoolz.svg)](https://badge.fury.io/py/iotoolz)\n[![Build Status](https://travis-ci.com/e2fyi/iotoolz.svg?branch=master)](https://travis-ci.com/github/e2fyi/iotoolz)\n[![Coverage Status](https://coveralls.io/repos/github/e2fyi/iotoolz/badge.svg?branch=master)](https://coveralls.io/github/e2fyi/iotoolz?branch=master)\n[![Documentation Status](https://readthedocs.org/projects/iotoolz/badge/?version=latest)](https://iotoolz.readthedocs.io/en/latest/?badge=latest)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)\n[![Downloads](https://pepy.tech/badge/iotoolz/month)](https://pepy.tech/project/iotoolz)\n\n`iotoolz` is an improvement over `e2fyi-utils` and is inspired partly by `toolz`.\n`iotoolz` is a lib to help provide a consistent dev-x for interacting with any IO resources.\nIt provides an abstract class `iotoolz.AbcStream` which mimics python\'s native `open`\nvery closely (with some additional parameters and methods such as `save`).\n\nAPI documentation can be found at [https://iotoolz.readthedocs.io/en/latest/](https://iotoolz.readthedocs.io/en/latest/).\n\nChange logs are available in [CHANGELOG.md](https://github.com/e2fyi/iotoolz/blob/master/CHANGELOG.md).\n\n> - Python 3.6 and above\n> - Licensed under [Apache-2.0](./LICENSE).\n\n## Supported streams\n\nCurrent the following streams are supported:\n\n- `iotoolz.FileStream`: wrapper over built-in `open` function (`file://`)\n- `iotoolz.TempStream`: in-memory stream that will rollover to disk (`tmp://`, `temp://`)\n- `iotoolz.HttpStream`: http or https stream implemented with `requests` (`http://`, `https://`)\n- `iotoolz.extensions.S3Stream`: s3 stream implemented with `boto3` (`s3://`, `s3a://`, `s3n://`)\n\n## Installation\n\n```bash\n# install the default packages only (most lite-weight)\npip install iotoolz\n\n# install dependencies for specific extension\npip install iotoolz[boto3]\n\n# install all the extras\npip install iotoolz[all]\n```\n\nAvailable extras:\n\n- `all`: All the optional dependencies\n- `boto3`: `boto3` for `iotoolz.extensions.S3Stream`\n- `minio`: TODO\n\n## Quickstart\n\nThe helper object `iotoolz.streams.stream_factory` is a default singleton of\n`iotoolz.streams.Streams` provided to support most of the common use cases.\n\n`iotoolz.streams.open_stream` (alias `iotoolz.streams.Stream`) and is a util method\nprovided by the singleton helper to create a stream object. This method accepts the same\narguments as python\'s `open` method with the following additional parameters:\n\n- `data`: optional str or bytes that will be passed into the stream\n- `fileobj`: optional file-like object which will be copied into the stream\n- `content_type`: optional mime type information to describe the stream (e.g. application/json)\n- `inmem_size`: determines how much memory to allocate to the stream before rolling over to local file system. Defaults to no limits (may result in MemoryError).\n- `schema_kwargs`: optional mapping of schemas to their default kwargs.\n\n### Basic Setup\n\n```py\nfrom iotoolz.streams import (\n    set_schema_kwargs,\n    set_buffer_rollover_size,\n)\n\n# set params to pass to the Stream obj handling https\n# i.e. HttpStream (implemented with requests)\nset_schema_kwargs(\n    "https",\n    verify=False,  # do not verify ssl cert\n    use_post=True  # use POST instead of PUT when writing to https\n)\n\n# use a custom client for S3Stream (via boto3)\nset_schema_kwargs(\n    "s3",\n    client=boto3.client(\n        "s3",\n        aws_access_key_id=ACCESS_KEY,\n        aws_secret_access_key=SECRET_KEY,\n        aws_session_token=SESSION_TOKEN,\n    )\n)\n\n# buffer will rollover to disk if the data is more than 100 MB\n# (default is everything is in-memory - may result in memory error)\nset_buffer_rollover_size(10**8)\n```\n\n### Opening streams\n\nYou can open any stream just like python\'s built-in `open` method.\n\n```py\nimport pandas\n\nfrom iotoolz import open_stream\n\n# print line by line some data in from a https endpoint\n# and do not verify the ssl cert of the https endpoint\nwith open_stream(\n    "https://foo/bar/data.txt",\n    mode="r",\n    schema_kwargs={"https": {"verify": False}}\n) as stream:\n    for line in stream:\n        print(line)\n\n# POST some binary content to a http endpoint (default is PUT)\nwith open_stream("https://foo.bar/api/data", "wb", use_post=True) as stream:\n    stream.write(b"hello world")\n\n# Copying a local file to s3\nwith open_stream("path/to/data.csv", "r") as csv_source,\n     open_stream("s3://bucket/foobar.txt?StorageClass=STANDARD", "w") as s3_sink:\n    # pipe content in csv_source to tmpsink\n    csv_source.pipe(s3_sink)\n\n# load to pandas dataframe from s3 fileobj\nwith open_stream("s3://bucket/foobar.csv", "r") as csv:\n    df = pd.read_csv(csv)\n\n```\n\n## TempStream\n\n`TempStream` is a stream can functions like a virtual file system in memory.\n\n```py\nimport gc\n\nfrom iotoolz import Stream, exists, glob, iter_dir\n\n# this stream can be garbage collected\nStream("tmp://foo/bar/data.txt", data="foobar")\n\n# True if not gc yet, False if already gc\nexists("tmp://foo/bar/data.txt")\n\n# force gc\ngc.collect()\n# will not exist\nexists("tmp://foo/bar/data.txt")\n\n# create temp stream with strong ref (hence will not be gc)\ns1 = Stream("tmp://foo/bar/data.txt", data="foobar")\ns2 = Stream("tmp://foo/example.txt", data="...")\n\n# returns s1 and s2\niter_dir("tmp://foo/")\n\n# returns s1 only\nglob("tmp://foo/bar/*.txt")\n```\n\n## Stream-like operations\n\n`Stream` is an alias of `open_stream`, both methods return a concrete `AbcStream` object.\nYou can treat the object as both a "file-like" and "stream-like" object - i.e. you can\nread, write, seek, flush, close the object.\n\n> NOTE\n>\n> By default, the underlying buffer is in-memory. You can enable rollover to disk by\n> passing the `inmem_size` arg to the method, or update the default `inmem_size` value\n> with the `iotoolz.streams.set_buffer_rollover_size` method.\n\n```py\nfrom iotoolz import open_stream, Stream, set_buffer_rollover_size\n\n# `Stream` is an alias of `open_stream`\nassert open_stream == Stream\n\n# rollover to disk if data is over 100 MB\nset_buffer_rollover_size(10**8)\n\n# you can overwrite the default kwargs here also\nstream = Stream(\n    "path/to/data.txt",\n    mode="rw",  # you can both read and write to a stream\n)\n# stream is lazily evaluated, nothing will be buffered until you call some methods\n# that requires the data\ndata = stream.read()\n# will attempt to provide encoding and content_type if not provided when opening the stream\nprint(stream.encoding)\nprint(stream.content_type)\n# stream has the same interface as an IO object - i.e. u can seek, flush, close, etc\nstream.seek(5)  # go to offset 5 from start of buffer\nstream.write("replace with this text")\nstream.seek(0, whence=2)  # go to end of buffer\nstream.write("additional text after original eof")  # continue writing to the end of the buffer\nstream.save()  # flush save the entire buffer to the same dst location\nstream.close() # close the stream\n```\n\n## Path-like operations\n\n`exists`, `mkdir`, `iter_dir` and `glob` are path-like methods that are available to the\nstream object. These methods mimics their equivalent in `pathlib.Path` when appropriate.\n\n| method                       | supported streams                           | desc                                                                                       |\n| ---------------------------- | ------------------------------------------- | ------------------------------------------------------------------------------------------ |\n| `stats`                      | All Streams                                 | return the StreamInfo for an existing resource                                             |\n| `unlink`, `delete`, `remove` | All Streams                                 | Delete and remove the stream (except for `TempStream` where the buffer is cleared instead) |\n| `exists`                     | All Streams                                 | check if a stream points to an existing resource.                                          |\n| `mkdir`                      | `FileStream`                                | create a directory.                                                                        |\n| `rmdir`                      | `FileStream`, `TempStream`, and `S3Stream`, | remove recursively everything in the directory.                                            |\n| `iter_dir`                   | `FileStream`, `TempStream`, and `S3Stream`  | iterate thru the streams in the directory.                                                 |\n| `glob`                       | `FileStream`, `TempStream`, and `S3Stream`  | iterate thru the streams in the directory that match a pattern.                            |\n\n```py\nimport itertools\n\nfrom iotoolz import Stream, mkdir, iter_dir, glob, exists\n\n# similar to \'mkdir -p\'\nmkdir("path/to/folder", parents=True, exist_ok=True)\nStream("path/to/folder").mkdir(parents=True, exist_ok=True)\n\n# list object in an s3 bucket\niter_dir("s3://bucket/prefix/")\nfor stream in Stream("s3://bucket/prefix/").iter_dir():\n    print(stream.uri)\n\n# find s3 objects with a specific pattern\nglob("s3://bucket/prefix/*txt")\nfor stream in Stream("s3://bucket/prefix/").glob("*.txt"):\n    print(stream.uri)\n\n# exists\nexists("s3://bucket/prefix/foo.txt")\n\n# stats\ninfo = stats("s3://bucket/prefix/foo.txt")\nprint(info.name)\nprint(info.content_type)\nprint(info.encoding)\nprint(info.last_modified)\nprint(info.etag)\nprint(info.extras)\n\n# delete resource\nunlink("s3://bucket/prefix/foo.txt")\n\n# rm all key with prefix\nrmdir("s3://bucket/prefix/")\n\n```\n\n## Piping streams\n\n`pipe` is method to push data to a sink (similar to NodeJS stream except it has no\nwatermark or buffering).\n\n```py\nfrom  iotoolz.streams import open_stream\n\nlocal_file = open_stream(\n    "path/to/google.html", content_type="text/html", mode="w"\n)\ntemp_file = open_stream(\n    "tmp://google.html", content_type="text/html", mode="wb"\n)\n\n# when source is closed, all sinks will be closed also\nwith open_stream("https://google.com") as source:\n    # writes to a temp file then to a local file in sequence\n    source.pipe(temp_file).pipe(local_file)\n\n\nlocal_file2 = open_stream(\n    "path/to/google1.html", content_type="text/html", mode="w"\n)\nlocal_file3 = open_stream(\n    "path/to/google2.html", content_type="text/html", mode="w"\n)\n\n# when source is closed, all sinks will be closed also\nwith open_stream("tmp://foo_src", mode="w") as source:\n    # writes in a fan shape manner\n    source.pipe(local_file2)\n    source.pipe(local_file3)\n\n    source.write("hello world")\n```\n\n> TODO support transform streams so that pipe can be more useful\n',
    'author': 'eterna2',
    'author_email': 'eterna2@hotmail.com',
    'maintainer': 'eterna2',
    'maintainer_email': 'eterna2@hotmail.com',
    'url': 'https://github.com/e2fyi/iotoolz',
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'extras_require': extras_require,
    'python_requires': '>=3.6,<4',
}


setup(**setup_kwargs)
