# -*- coding: utf-8 -*-
from setuptools import setup

package_dir = \
{'': 'src'}

packages = \
['airfly',
 'airfly._vendor',
 'airfly._vendor.airflow',
 'airfly._vendor.airflow.contrib',
 'airfly._vendor.airflow.contrib.operators',
 'airfly._vendor.airflow.contrib.sensors',
 'airfly._vendor.airflow.decorators',
 'airfly._vendor.airflow.example_dags',
 'airfly._vendor.airflow.models',
 'airfly._vendor.airflow.operators',
 'airfly._vendor.airflow.providers',
 'airfly._vendor.airflow.providers.amazon',
 'airfly._vendor.airflow.providers.amazon.aws',
 'airfly._vendor.airflow.providers.amazon.aws.operators',
 'airfly._vendor.airflow.providers.amazon.aws.sensors',
 'airfly._vendor.airflow.providers.amazon.aws.transfers',
 'airfly._vendor.airflow.providers.apache',
 'airfly._vendor.airflow.providers.apache.cassandra',
 'airfly._vendor.airflow.providers.apache.cassandra.sensors',
 'airfly._vendor.airflow.providers.apache.druid',
 'airfly._vendor.airflow.providers.apache.druid.operators',
 'airfly._vendor.airflow.providers.apache.druid.transfers',
 'airfly._vendor.airflow.providers.apache.hdfs',
 'airfly._vendor.airflow.providers.apache.hdfs.sensors',
 'airfly._vendor.airflow.providers.apache.hive',
 'airfly._vendor.airflow.providers.apache.hive.operators',
 'airfly._vendor.airflow.providers.apache.hive.sensors',
 'airfly._vendor.airflow.providers.apache.hive.transfers',
 'airfly._vendor.airflow.providers.apache.pig',
 'airfly._vendor.airflow.providers.apache.pig.operators',
 'airfly._vendor.airflow.providers.apache.spark',
 'airfly._vendor.airflow.providers.apache.spark.operators',
 'airfly._vendor.airflow.providers.apache.sqoop',
 'airfly._vendor.airflow.providers.apache.sqoop.operators',
 'airfly._vendor.airflow.providers.celery',
 'airfly._vendor.airflow.providers.celery.sensors',
 'airfly._vendor.airflow.providers.cncf',
 'airfly._vendor.airflow.providers.cncf.kubernetes',
 'airfly._vendor.airflow.providers.cncf.kubernetes.operators',
 'airfly._vendor.airflow.providers.databricks',
 'airfly._vendor.airflow.providers.databricks.operators',
 'airfly._vendor.airflow.providers.datadog',
 'airfly._vendor.airflow.providers.datadog.sensors',
 'airfly._vendor.airflow.providers.dingding',
 'airfly._vendor.airflow.providers.dingding.operators',
 'airfly._vendor.airflow.providers.discord',
 'airfly._vendor.airflow.providers.discord.operators',
 'airfly._vendor.airflow.providers.docker',
 'airfly._vendor.airflow.providers.docker.operators',
 'airfly._vendor.airflow.providers.ftp',
 'airfly._vendor.airflow.providers.ftp.sensors',
 'airfly._vendor.airflow.providers.google',
 'airfly._vendor.airflow.providers.google.cloud',
 'airfly._vendor.airflow.providers.google.cloud.operators',
 'airfly._vendor.airflow.providers.google.cloud.sensors',
 'airfly._vendor.airflow.providers.google.cloud.transfers',
 'airfly._vendor.airflow.providers.google.suite',
 'airfly._vendor.airflow.providers.google.suite.transfers',
 'airfly._vendor.airflow.providers.grpc',
 'airfly._vendor.airflow.providers.grpc.operators',
 'airfly._vendor.airflow.providers.http',
 'airfly._vendor.airflow.providers.http.operators',
 'airfly._vendor.airflow.providers.http.sensors',
 'airfly._vendor.airflow.providers.imap',
 'airfly._vendor.airflow.providers.imap.sensors',
 'airfly._vendor.airflow.providers.jdbc',
 'airfly._vendor.airflow.providers.jdbc.operators',
 'airfly._vendor.airflow.providers.jenkins',
 'airfly._vendor.airflow.providers.jenkins.operators',
 'airfly._vendor.airflow.providers.jira',
 'airfly._vendor.airflow.providers.jira.operators',
 'airfly._vendor.airflow.providers.jira.sensors',
 'airfly._vendor.airflow.providers.microsoft',
 'airfly._vendor.airflow.providers.microsoft.azure',
 'airfly._vendor.airflow.providers.microsoft.azure.operators',
 'airfly._vendor.airflow.providers.microsoft.azure.sensors',
 'airfly._vendor.airflow.providers.microsoft.azure.transfers',
 'airfly._vendor.airflow.providers.microsoft.mssql',
 'airfly._vendor.airflow.providers.microsoft.mssql.operators',
 'airfly._vendor.airflow.providers.microsoft.winrm',
 'airfly._vendor.airflow.providers.microsoft.winrm.operators',
 'airfly._vendor.airflow.providers.mongo',
 'airfly._vendor.airflow.providers.mongo.sensors',
 'airfly._vendor.airflow.providers.mysql',
 'airfly._vendor.airflow.providers.mysql.operators',
 'airfly._vendor.airflow.providers.mysql.transfers',
 'airfly._vendor.airflow.providers.opsgenie',
 'airfly._vendor.airflow.providers.opsgenie.operators',
 'airfly._vendor.airflow.providers.oracle',
 'airfly._vendor.airflow.providers.oracle.operators',
 'airfly._vendor.airflow.providers.oracle.transfers',
 'airfly._vendor.airflow.providers.papermill',
 'airfly._vendor.airflow.providers.papermill.operators',
 'airfly._vendor.airflow.providers.postgres',
 'airfly._vendor.airflow.providers.postgres.operators',
 'airfly._vendor.airflow.providers.qubole',
 'airfly._vendor.airflow.providers.qubole.operators',
 'airfly._vendor.airflow.providers.qubole.sensors',
 'airfly._vendor.airflow.providers.redis',
 'airfly._vendor.airflow.providers.redis.operators',
 'airfly._vendor.airflow.providers.redis.sensors',
 'airfly._vendor.airflow.providers.segment',
 'airfly._vendor.airflow.providers.segment.operators',
 'airfly._vendor.airflow.providers.sftp',
 'airfly._vendor.airflow.providers.sftp.operators',
 'airfly._vendor.airflow.providers.sftp.sensors',
 'airfly._vendor.airflow.providers.slack',
 'airfly._vendor.airflow.providers.slack.operators',
 'airfly._vendor.airflow.providers.snowflake',
 'airfly._vendor.airflow.providers.snowflake.operators',
 'airfly._vendor.airflow.providers.sqlite',
 'airfly._vendor.airflow.providers.sqlite.operators',
 'airfly._vendor.airflow.providers.ssh',
 'airfly._vendor.airflow.providers.ssh.operators',
 'airfly._vendor.airflow.providers.vertica',
 'airfly._vendor.airflow.providers.vertica.operators',
 'airfly._vendor.airflow.sensors',
 'airfly._vendor.airflow.serialization',
 'airfly._vendor.airflow.utils',
 'airfly._vendor.airflow.utils.log',
 'airfly.cli',
 'airfly.model']

package_data = \
{'': ['*']}

install_requires = \
['astor>=0.8.1,<0.9.0',
 'attrs<21',
 'black>=21.7b0,<22.0',
 'cattrs<1.7.0',
 'click>=8.0.1,<9.0.0',
 'isort<5.9',
 'networkx>=2.6.2,<3.0.0',
 'regex>=2021.9.24,<2022.0.0']

entry_points = \
{'console_scripts': ['airfly = airfly.cli.main:main']}

setup_kwargs = {
    'name': 'airfly',
    'version': '0.6.1',
    'description': "Auto generate Airflow's dag.py on the fly",
    'long_description': '![](https://github.com/ryanchao2012/airfly/actions/workflows/airfly-run-unittests.yml/badge.svg)\n![](https://img.shields.io/pypi/v/airfly.svg)\n![](https://img.shields.io/pypi/pyversions/airfly)\n![](https://img.shields.io/github/license/ryanchao2012/airfly)\n\n\n# AirFly: Auto Generate Airflow\'s `dag.py` On The Fly\n\nPipeline management is essential for data operation in company, many engineering teams rely on tools like Airflow to help them organize workflows, such as ETL, data analytic jobs or machine learning projects.\n\nAirflow provides rich extensibility to let developers arrange workloads into a sequence of "operators", then they declare the task dependencies within a `DAG` context while writing the `dag.py` file. \n\nAs workflow grows progressively, the increasing complexity of task relations prones to messing up the dag structure, leads to decrease of code maintainability, especially in collaborative scenarios.\n\n`airfly` tries to mitigate such pain-points and brings automation to this development life cycle, it assumes all tasks are managed in certain python module, developers specify the dependencies while defining the task objects. During deployment, `airfly` can resolve the dependency tree and automatically build the `dag.py` for you.\n\n## Install\n\nDownload `airfly` from PyPi\n\n```\n$ pip install airfly\n\n$ airfly --help\nUsage: airfly [OPTIONS]\n\nOptions:\n  --version                   Show version and exit.\n  -n, --name TEXT             Assign to DAG id.\n  -m, --modname TEXT          Name of the module to search tasks for building\n                              the task dependency tree and using it to\n                              generate the airflow DAG file.\n  -p, --path TEXT             Insert into "sys.path" to include certain\n                              modules, multi-value is allowed.\n  -e, --exclude-pattern TEXT  Exclude the tasks from the dependency tree if\n                              their __qualname__ get matched with this regex\n                              pattern.\n  -i, --includes TEXT         Paths of python files, the codes within will be\n                              included in the output DAG file, multi-value is\n                              allowed.\n  -d, --dag-params TEXT       Parameters to construct DAG object, defined by a\n                              dictionary in a python file. Pass this option\n                              with <python-file>:<variable> form, the\n                              <variable> should be the dictionary which will\n                              be passed to DAG as keyword arguments.\n  --help                      Show this message and exit.\n\n```\n\n## Usage\n\n`airfly` expects the implementations are populated in a Python module(or package), the task dependencies are declared by assigning `upstreams` and `downstreams` attributes to each object. The task objects are actually wrappers for Airflow operators, when `airfly` walks through the entire module, all tasks are discovered and collected, the dependency tree and the `DAG` context are automatically built, with some `ast` helpers, `airfly` can wrap all these information, convert them into python code, and finally save them to `dag.py`.\n\n<img src="https://github.com/ryanchao2012/airfly/blob/main/assets/layout.png?raw=true" width="500">\n\n\n### Wrap Airflow operator with `AirflowTask`\n\nIn order to do codegen, collect the operator\'s metadata into a `AirflowTask` subclass as following(see [demo](https://github.com/ryanchao2012/airfly/blob/main/examples/tutorial/demo.py)):\n\n```python\n# in demo.py\nfrom airfly.model.airflow import AirflowTask\n\n\nclass print_date(AirflowTask):\n    operator_class = "BashOperator" \n    params = dict(bash_command="date")\n```\n\n* `operator_class` specifies the class of the Airflow operator.\n* The class name(`print_date`) will be mapped to `task_id` to the applied operator after code generation.\n* `params` will be passed to operator as keyword argument.\n\n\n### Declare task dependency\n\nUse `upstreams` and `downstreams` to specify task dependencies.\n\n```python\n# in demo.py\n\nfrom textwrap import dedent\n\n\ntemplated_command = dedent(\n    """\n{% for i in range(5) %}\n    echo "{{ ds }}"\n    echo "{{ macros.ds_add(ds, 7)}}"\n    echo "{{ params.my_param }}"\n{% endfor %}\n"""\n)\n\nclass templated(AirflowTask):\n    operator_class = "BashOperator"\n    params = dict(depends_on_past=False,\n                  bash_command=templated_command,\n                  params={"my_param": "Parameter I passed in"})\n\n\nclass sleep(AirflowTask):\n    operator_class = "BashOperator"\n    params = dict(depends_on_past=False, \n                  bash_command="sleep 5",\n                  retries=3)\n\n    upstreams = print_date\n    downstreams = (templated,)\n```\n\n\n### Generate the `dag.py` file\nWith commandline interface:\n```\n$ airfly --name demo_dag --modname demo > dag.py\n```\n\nThe outputs in `dag.py`:\n\n```python\n# This file is auto-generated by airfly 0.6.0\nfrom airflow.models import DAG\nfrom airflow.operators.bash import BashOperator\n\nwith DAG("demo_dag") as dag:\n    demo_print_date = BashOperator(task_id="demo.print_date", bash_command="date")\n    demo_sleep = BashOperator(\n        task_id="demo.sleep", depends_on_past=False, bash_command="sleep 5", retries=3\n    )\n    demo_templated = BashOperator(\n        task_id="demo.templated",\n        depends_on_past=False,\n        bash_command="""\n{% for i in range(5) %}\n    echo "{{ ds }}"\n    echo "{{ macros.ds_add(ds, 7)}}"\n    echo "{{ params.my_param }}"\n{% endfor %}\n""",\n        params={"my_param": "Parameter I passed in"},\n    )\n    demo_print_date >> demo_sleep\n    demo_sleep >> demo_templated\n```\n\n\n## Inject parameters to `DAG`\n\nIf any additional arguments are needed, write and manage those configurations in a python file(see [demo](https://github.com/ryanchao2012/airfly/blob/main/examples/tutorial/params.py)), `airfly` can pass them to `DAG` during codegen.\n\n```python\n# in params.py\n\nfrom datetime import timedelta\n\nfrom airflow.utils.dates import days_ago\n\ndefault_args = {\n    "owner": "airflow",\n    "depends_on_past": False,\n    "email": ["airflow@example.com"],\n    "email_on_failure": False,\n    "email_on_retry": False,\n    "retries": 1,\n    "retry_delay": timedelta(minutes=5),\n    # \'queue\': \'bash_queue\',\n    # \'pool\': \'backfill\',\n    # \'priority_weight\': 10,\n    # \'end_date\': datetime(2016, 1, 1),\n    # \'wait_for_downstream\': False,\n    # \'dag\': dag,\n    # \'sla\': timedelta(hours=2),\n    # \'execution_timeout\': timedelta(seconds=300),\n    # \'on_failure_callback\': some_function,\n    # \'on_success_callback\': some_other_function,\n    # \'on_retry_callback\': another_function,\n    # \'sla_miss_callback\': yet_another_function,\n    # \'trigger_rule\': \'all_success\'\n}\n\ndag_kwargs = dict(\n    default_args=default_args,\n    description="A simple tutorial DAG",\n    schedule_interval=timedelta(days=1),\n    start_date=days_ago(2),\n    tags=["example"],\n)\n```\n\nInject those arguments with `--dag-params` option:\n```\n$ airfly --name demo_dag --modname demo --dag-params params.py:dag_kwargs > dag.py\n```\n\nThe outputs in `dag.py`:\n```python\n# This file is auto-generated by airfly 0.6.0\nfrom datetime import timedelta\n\nfrom airflow.models import DAG\nfrom airflow.operators.bash import BashOperator\nfrom airflow.utils.dates import days_ago\n\n# >>>>>>>>>> Include from \'params.py\'\ndefault_args = {\n    "owner": "airflow",\n    "depends_on_past": False,\n    "email": ["airflow@example.com"],\n    "email_on_failure": False,\n    "email_on_retry": False,\n    "retries": 1,\n    "retry_delay": timedelta(minutes=5),\n}\ndag_kwargs = dict(\n    default_args=default_args,\n    description="A simple tutorial DAG",\n    schedule_interval=timedelta(days=1),\n    start_date=days_ago(2),\n    tags=["example"],\n)\n# <<<<<<<<<< End of code insertion\nwith DAG("demo_dag", **dag_kwargs) as dag:\n    demo_print_date = BashOperator(task_id="demo.print_date", bash_command="date")\n    demo_sleep = BashOperator(\n        task_id="demo.sleep", depends_on_past=False, bash_command="sleep 5", retries=3\n    )\n    demo_templated = BashOperator(\n        task_id="demo.templated",\n        depends_on_past=False,\n        bash_command="""\n{% for i in range(5) %}\n    echo "{{ ds }}"\n    echo "{{ macros.ds_add(ds, 7)}}"\n    echo "{{ params.my_param }}"\n{% endfor %}\n""",\n        params={"my_param": "Parameter I passed in"},\n    )\n    demo_print_date >> demo_sleep\n    demo_sleep >> demo_templated\n```\n\n`airfly` wraps required information including variables and imports into output python script, and pass the specified value to `DAG` object.\n\n\n## Exclude tasks from codegen\nBy passing `--exclude-pattern` to match any unwanted objects with their `__qualname__`. then filter them out.\n```\n$ airfly --name demo_dag --modname demo --exclude-pattern templated > dag.py\n```\n\nThe outputs in `dag.py`:\n\n```python\n# This file is auto-generated by airfly 0.6.0\nfrom airflow.models import DAG\nfrom airflow.operators.bash import BashOperator\n\nwith DAG("demo_dag") as dag:\n    demo_print_date = BashOperator(task_id="demo.print_date", bash_command="date")\n    demo_sleep = BashOperator(\n        task_id="demo.sleep", depends_on_past=False, bash_command="sleep 5", retries=3\n    )\n    demo_print_date >> demo_sleep\n```\n\nThe `templated` task is gone.\n\n\n## Examples\n\nPlease visit [examples](https://github.com/ryanchao2012/airfly/blob/main/examples) to explore more use cases.\n<a href="https://github.com/ryanchao2012/airfly/blob/main/examples"><img src="https://github.com/ryanchao2012/airfly/blob/main/assets/view.png?raw=true" width="500"></img></a>',
    'author': 'ryanchao2012',
    'author_email': 'ryanchao2012@gmail.com',
    'maintainer': None,
    'maintainer_email': None,
    'url': 'https://github.com/ryanchao2012/airfly',
    'package_dir': package_dir,
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'entry_points': entry_points,
    'python_requires': '>=3.7,<4.0',
}


setup(**setup_kwargs)
