# -*- coding: utf-8 -*-
from setuptools import setup

package_dir = \
{'': 'src'}

packages = \
['stairlight',
 'stairlight.source',
 'stairlight.source.dbt',
 'stairlight.source.file',
 'stairlight.source.gcs',
 'stairlight.source.redash',
 'stairlight.source.s3']

package_data = \
{'': ['*'], 'stairlight.source.redash': ['sql/*']}

install_requires = \
['Jinja2>=2.10.3', 'PyYAML>=5.0']

extras_require = \
{':extra == "s3"': ['boto3-stubs[s3]>=1.24.17,<2.0.0'],
 'dbt-bigquery': ['dbt-core>=1.1,<2.0', 'dbt-bigquery>=1.1,<2.0'],
 'gcs': ['google-cloud-storage>=1.28.1,<2.0.0'],
 'redash': ['psycopg2>=2.9.3,<3.0.0', 'SQLAlchemy>=1.4.31,<2.0.0'],
 's3': ['boto3>=1.24.14,<2.0.0']}

entry_points = \
{'console_scripts': ['stairlight = stairlight.cli:main']}

setup_kwargs = {
    'name': 'stairlight',
    'version': '0.6.0',
    'description': 'An end-to-end data lineage tool',
    'long_description': '<div align="center">\n  <img src="https://raw.githubusercontent.com/tosh2230/stairlight/main/img/stairlight_white.png" width="400" alt="Stairlight">\n</div>\n\n-----------------\n\n# Stairlight\n\n[![PyPi Version](https://img.shields.io/pypi/v/stairlight.svg?style=flat-square&logo=PyPi)](https://pypi.org/project/stairlight/)\n[![PyPi License](https://img.shields.io/pypi/l/stairlight.svg?style=flat-square)](https://pypi.org/project/stairlight/)\n[![PyPi Python Versions](https://img.shields.io/pypi/pyversions/stairlight.svg?style=flat-square)](https://pypi.org/project/stairlight/)\n[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/psf/black)\n[![CI](https://github.com/tosh2230/stairlight/actions/workflows/ci.yml/badge.svg)](https://github.com/tosh2230/stairlight/actions/workflows/ci.yml)\n\nAn end-to-end data lineage tool, detects table dependencies by SQL SELECT statements.\n\n<div align="left">\n  <img src="https://raw.githubusercontent.com/tosh2230/stairlight/main/img/drawio/concepts.drawio.png" width="800" alt="concepts">\n</div>\n\n## Supported Data Sources\n\n| Data Source | Remarks |\n| --- | --- |\n| Local file system | With Python Pathlib module |\n| [Amazon S3](https://aws.amazon.com/s3/) | |\n| [Google Cloud Storage](https://cloud.google.com/storage) | Also available for [Google Cloud Composer](https://cloud.google.com/composer) |\n| [dbt](https://www.getdbt.com/) - [Google BigQuery](https://cloud.google.com/bigquery) | Using `dbt compile` command internally |\n| [Redash](https://redash.io/) | |\n\n## Installation\n\nThis package is distributed on [PyPI](https://pypi.org/project/stairlight/).\n\n```sh\n# The base package is for local file system only.\n$ pip install stairlight\n\n# Set extras when detecting from other data sources.\n# e.g. Amazon S3 and Google Cloud Storage\n$ pip install "stairlight[s3, gcs]"\n```\n\n| Data Source | TemplateSourceType | Extra |\n| --- | --- | --- |\n| Local file system | File | - |\n| Amazon S3 | S3 | s3 |\n| Google Cloud Storage | GCS | gcs |\n| dbt - Google Bigquery | dbt | dbt-bigquery |\n| Redash | Redash | redash |\n\n## Getting Started\n\nThere are 3 steps to use.\n\n```sh\n# Step 1: Initialize and set data location settings\n$ stairlight init\n\'./stairlight.yaml\' has created.\nPlease edit it to set your data sources.\n\n# Step 2: Map your SQL statements and tables, and add metadata\n$ stairlight map\n\'./mapping_yyyyMMddhhmmss.yaml\' has created.\nPlease map undefined tables and parameters, and append to your latest configuration file.\n\n# Step 3: Get a table dependency map\n$ stairlight\n```\n\n## Description\n\n### Input\n\n- SQL SELECT statements\n- Configuration YAML files\n    - stairlight.yaml: SQL statements locations and include/exclude conditions.\n    - mapping.yaml: Mapping SQL statements and tables.\n\n### Output\n\n- JSON dependency map\n\n    <details>\n\n    <summary>Example</summary>\n\n    ```json\n    {\n      "PROJECT_d.DATASET_e.TABLE_f": {\n        "PROJECT_j.DATASET_k.TABLE_l": {\n          "TemplateSourceType": "File",\n          "Key": "tests/sql/main/one_line_2.sql",\n          "Uri": "/foo/bar/stairlight/tests/sql/main/one_line_2.sql",\n          "Lines": [\n            {\n              "LineNumber": 1,\n              "LineString": "SELECT * FROM PROJECT_j.DATASET_k.TABLE_l WHERE 1 = 1"\n            }\n          ]\n        },\n        "PROJECT_C.DATASET_C.TABLE_C": {\n          "TemplateSourceType": "GCS",\n          "Key": "sql/cte/cte_multi_line.sql",\n          "Uri": "gs://stairlight/sql/cte/cte_multi_line.sql",\n          "Lines": [\n            {\n              "LineNumber": 6,\n              "LineString": "        PROJECT_C.DATASET_C.TABLE_C"\n            }\n          ],\n          "BucketName": "stairlight",\n          "Labels": {\n            "Source": "gcs",\n            "Test": "b"\n          }\n        }\n      },\n      "AggregateSales": {\n        "PROJECT_e.DATASET_e.TABLE_e": {\n          "TemplateSourceType": "Redash",\n          "Key": 5,\n          "Uri": "AggregateSales",\n          "Lines": [\n            {\n              "LineNumber": 1,\n              "LineString": "SELECT service, SUM(total_amount) FROM PROJECT_e.DATASET_e.TABLE_e GROUP BY service"\n            }\n          ],\n          "DataSourceName": "BigQuery",\n          "Labels": {\n            "Category": "Sales"\n          }\n        }\n      },\n      "dummy.dummy.example_b": {\n        "PROJECT_t.DATASET_t.TABLE_t": {\n          "TemplateSourceType": "dbt",\n          "Key": "tests/dbt/project_01/target/compiled/project_01/models/b/example_b.sql",\n          "Uri": "/foo/bar/stairlight/tests/dbt/project_01/target/compiled/project_01/models/b/example_b.sql",\n          "Lines": [\n            {\n              "LineNumber": 1,\n              "LineString": "select * from PROJECT_t.DATASET_t.TABLE_t where value_a = 0 and value_b = 0"\n            }\n          ]\n        }\n      },\n      "PROJECT_as.DATASET_bs.TABLE_cs": {\n        "PROJECT_A.DATASET_A.TABLE_A": {\n          "TemplateSourceType": "S3",\n          "Key": "sql/one_line/one_line.sql",\n          "Uri": "s3://stairlight/sql/one_line/one_line.sql",\n          "Lines": [\n            {\n              "LineNumber": 1,\n              "LineString": "SELECT * FROM PROJECT_A.DATASET_A.TABLE_A WHERE 1 = 1"\n            }\n          ],\n          "BucketName": "stairlight",\n          "Labels": {\n            "Source": null,\n            "Test": "a"\n          }\n        }\n      }\n    }\n    ```\n\n    </details>\n\n### Collecting patterns\n\n#### Centralization\n\n<div align="left">\n  <img src="https://raw.githubusercontent.com/tosh2230/stairlight/main/img/drawio/centralization.drawio.png" width="800" alt="centralization">\n</div>\n\n#### Agents\n\n<div align="left">\n  <img src="https://raw.githubusercontent.com/tosh2230/stairlight/main/img/drawio/agents.drawio.png" width="800" alt="agents">\n</div>\n\n## Configuration\n\nConfiguration files can be found [here](https://github.com/tosh2230/stairlight/tree/main/tests/config), used for unit testing in CI.\n\n### stairlight.yaml\n\n\'stairlight.yaml\' is for setting up Stairlight itself. It is responsible for specifying SQL statements to be read.\n\n`init` command creates a template of stairlight.yaml.\n\n  <details>\n\n  <summary>Example</summary>\n\n  ```yaml\n  Include:\n    - TemplateSourceType: File\n      FileSystemPath: "./tests/sql"\n      Regex: ".*/*.sql$"\n      DefaultTablePrefix: "PROJECT_A"\n    - TemplateSourceType: GCS\n      ProjectId: null\n      BucketName: stairlight\n      Regex: "^sql/.*/*.sql$"\n      DefaultTablePrefix: "PROJECT_A"\n    - TemplateSourceType: Redash\n      DatabaseUrlEnvironmentVariable: REDASH_DATABASE_URL\n      DataSourceName: BigQuery\n      QueryIds:\n        - 1\n        - 3\n        - 5\n    - TemplateSourceType: dbt\n      ProjectDir: tests/dbt/project_01\n      ProfilesDir: tests/dbt\n      Vars:\n        key_a: value_a\n        key_b: value_b\n    - TemplateSourceType: S3\n      BucketName: stairlight\n      Regex: "^sql/.*/*.sql$"\n      DefaultTablePrefix: "PROJECT_A"\n  Exclude:\n    - TemplateSourceType: File\n      Regex: "main/exclude.sql$"\n  Settings:\n    MappingPrefix: "mapping"\n  ```\n\n  </details>\n\n### mapping.yaml\n\n\'mapping.yaml\' is used to define relationships between input SELECT statements and table names.\n\n`map` command creates a template of mapping.yaml, based on the configuration of stairlight.yaml.\n\n  <details>\n\n  <summary>Example</summary>\n\n  ```yaml\n  Global:\n    Parameters:\n      DESTINATION_PROJECT: stairlight\n      params:\n        PROJECT: 1234567890\n        DATASET: public\n        TABLE: taxirides\n  Mapping:\n    - TemplateSourceType: File\n      FileSuffix: "tests/sql/main/union_same_table.sql"\n      Tables:\n        - TableName: "test_project.beam_streaming.taxirides_aggregation"\n          Parameters:\n            params:\n              source_table: source\n              destination_table: destination\n          IgnoreParameters:\n            - execution_date.add(days=1).isoformat()\n    - TemplateSourceType: GCS\n      Uri: "gs://stairlight/sql/one_line/one_line.sql"\n      Tables:\n        - TableName: "PROJECT_a.DATASET_b.TABLE_c"\n    - TemplateSourceType: Redash\n      QueryId: 5\n      DataSourceName: metadata\n      Tables:\n        - TableName: New Query\n          Parameters:\n            table: dashboards\n          Labels:\n            Category: Redash test\n    - TemplateSourceType: dbt\n      ProjectName: project_01\n      FileSuffix: tests/dbt/project_01/target/compiled/project_01/models/example/my_first_dbt_model.sql\n      Tables:\n        - TableName: dummy.dummy.my_first_dbt_model\n    - TemplateSourceType: S3\n      Uri: "s3://stairlight/sql/one_line/one_line.sql"\n      Tables:\n        - TableName: "PROJECT_as.DATASET_bs.TABLE_cs"\n  Metadata:\n    - TableName: "PROJECT_A.DATASET_A.TABLE_A"\n      Labels:\n        Source: Null\n        Test: a\n  ```\n\n  </details>\n\n#### Global Section\n\nThis section is for global configurations.\n\n`Parameters` attribute is used to set common parameters. If conflicts has occurred with `Parameters` attributes in mapping section, mapping section\'s parameters will be used in preference to global.\n\n#### Mapping Section\n\nMapping section is used to define relationships between input SELECT statements and tables that created as a result of query execution.\n\n`Parameters` attribute allows you to reflect settings in [jinja](https://jinja.palletsprojects.com/) template variables embedded in statements. If multiple settings are applied to a statement using jinja template, the statement will be read as if there were the same number of queries as the number of settings.\n\nIn contrast, `IgnoreParameters` attribute handles a list to ignore when rendering queries.\n\n#### Metadata Section\n\nThis section is mainly used to set metadata to tables appears only in queries.\n\n## Arguments and Options\n\n```txt\n$ stairlight --help\nusage: stairlight [-h] [-c CONFIG] [--save SAVE] [--load LOAD] {init,check,up,down} ...\n\nAn end-to-end data lineage tool, detects table dependencies by SQL SELECT statements.\nWithout positional arguments, return a table dependency map as JSON format.\n\npositional arguments:\n  {init,map,check,up,down}\n    init                create new Stairlight configuration file\n    map (check)         create new configuration file about undefined mappings\n    up                  return upstairs ( table | SQL file ) list\n    down                return downstairs ( table | SQL file ) list\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -c CONFIG, --config CONFIG\n                        set Stairlight configuration directory\n  -q, --quiet           keep silence\n  --save SAVE           A file path where map results will be saved.\n                        You can choose from local file system, GCS, S3.\n  --load LOAD           A file path where map results are saved.\n                        You can choose from local file system, GCS, S3.\n                        It can be specified multiple times.\n```\n\n### init\n\n`init` creates a new Stairlight configuration file.\n\n```txt\n$ stairlight init --help\nusage: stairlight init [-h] [-c CONFIG]\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -c CONFIG, --config CONFIG\n                        set Stairlight configuration directory.\n  -q, --quiet           keep silence\n```\n\n### map(check)\n\n`map` creates new configuration file about undefined mappings.`check` is an alias.\nThe option specification is the same as `init`.\n\n### up\n\n`up` outputs a list of tables or SQL files located upstream from the specified table.\n\n- Use table(`-t`, `--table`) or label(`-l`, `--label`) option to specify tables to search.\n- Recursive option(`-r`, `--recursive`) is set, Stairlight will find tables recursively and output as a list.\n- Verbose option(`-v`, `--verbose`) is set, Stairlight will add detailed information and output it as a dict.\n\n```txt\n$ stairlight up --help\nusage: stairlight up [-h] [-c CONFIG] [--save SAVE] [--load LOAD] (-t TABLE | -l LABEL) [-o {table,file}]\n                     [-v] [-r]\n\noptional arguments:\n  -h, --help            show this help message and exit\n  -c CONFIG, --config CONFIG\n                        set Stairlight configuration directory\n  -q, --quiet           keep silence\n  --save SAVE           A file path where map results will be saved.\n                        You can choose from local file system, GCS, S3.\n  --load LOAD           A file path where map results are saved.\n                        You can choose from local file system, GCS, S3.\n                        It can be specified multiple times.\n  -t TABLE, --table TABLE\n                        table names that Stairlight searches for, can be specified\n                        multiple times. e.g. -t PROJECT_a.DATASET_b.TABLE_c -t\n                        PROJECT_d.DATASET_e.TABLE_f\n  -l LABEL, --label LABEL\n                        labels set for the table in mapping configuration, can be specified multiple times.\n                        The separator between key and value should be a colon(:).\n                        e.g. -l key_1:value_1 -l key_2:value_2\n  -o {table,file}, --output {table,file}\n                        output type\n  -v, --verbose         return verbose results\n  -r, --recursive       search recursively\n```\n\n### down\n\n`down` outputs a list of tables or SQL files located downstream from the specified table.\nThe option specification is the same as `up`.\n\n## Use as a library\n\nStairlight can also be used as a library.\n\n[tosh2230/stairlight-app](https://github.com/tosh2230/stairlight-app) is a sample web application rendering table dependency graph with Stairlight, using Graphviz, Streamlit and Google Cloud Run.\n',
    'author': 'tosh2230',
    'author_email': 'rev.to12@gmail.com',
    'maintainer': None,
    'maintainer_email': None,
    'url': None,
    'package_dir': package_dir,
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'extras_require': extras_require,
    'entry_points': entry_points,
    'python_requires': '>=3.7.2,<4.0.0',
}


setup(**setup_kwargs)
