# -*- coding: utf-8 -*-
from setuptools import setup

packages = \
['csv_reconcile', 'csv_reconcile_dice']

package_data = \
{'': ['*']}

install_requires = \
['chardet>=4.0.0,<5.0.0',
 'cython>=0.29.21,<0.30.0',
 'flask-cors>=3.0.10,<4.0.0',
 'flask>=2.0,<3.0',
 'normality>=2.1.1,<3.0.0']

extras_require = \
{':python_version < "3.10"': ['importlib_metadata>=4.5.0,<5.0.0']}

entry_points = \
{'console_scripts': ['csv-reconcile = csv_reconcile:main'],
 'csv_reconcile.scorers': ['dice = csv_reconcile_dice']}

setup_kwargs = {
    'name': 'csv-reconcile',
    'version': '0.3.2',
    'description': 'OpenRefine reconciliation service backed by csv resource',
    'long_description': '-   <a href="#csv-reconcile" id="toc-csv-reconcile">CSV Reconcile</a>\n    -   <a href="#quick-start" id="toc-quick-start">Quick start</a>\n    -   <a href="#poetry" id="toc-poetry">Poetry</a>\n        -   <a href="#prerequesites" id="toc-prerequesites">Prerequesites</a>\n        -   <a href="#running" id="toc-running">Running</a>\n        -   <a href="#building" id="toc-building">Building</a>\n    -   <a href="#description" id="toc-description">Description</a>\n    -   <a href="#usage" id="toc-usage">Usage</a>\n        -   <a href="#initialization" id="toc-initialization">Initialization</a>\n        -   <a href="#running-the-service" id="toc-running-the-service">Running the\n            service</a>\n        -   <a href="#deprecated" id="toc-deprecated">Deprecated</a>\n    -   <a href="#common-configuration" id="toc-common-configuration">Common\n        configuration</a>\n    -   <a href="#built-in-preview-service"\n        id="toc-built-in-preview-service">Built-in preview service</a>\n    -   <a href="#scoring-plugins" id="toc-scoring-plugins">Scoring plugins</a>\n        -   <a href="#implementing" id="toc-implementing">Implementing</a>\n        -   <a href="#installing" id="toc-installing">Installing</a>\n        -   <a href="#using" id="toc-using">Using</a>\n        -   <a href="#known-plugins" id="toc-known-plugins">Known plugins</a>\n    -   <a href="#testing" id="toc-testing">Testing</a>\n        -   <a href="#tests-layout" id="toc-tests-layout">Tests layout</a>\n        -   <a href="#running-tests" id="toc-running-tests">Running tests</a>\n    -   <a href="#future-enhancements" id="toc-future-enhancements">Future\n        enhancements</a>\n\n# CSV Reconcile\n\nA [reconciliation service](https://github.com/reconciliation-api/specs)\nfor [OpenRefine](https://openrefine.org/) based on a CSV file similar to\n[reconcile-csv](http://okfnlabs.org/reconcile-csv/). This one is written\nin Python and has some more configurability.\n\n## Quick start\n\n-   Clone this repository\n\n-   Run the service\n\n        $ python -m venv venv                                             # create virtualenv\n        $ venv/bin/pip install csv-reconcile                              # install package\n        $ source venv/bin/activate                                        # activate virtual environment\n        (venv) $ csv-reconcile init sample/reps.tsv item itemLabel        # initialize the service\n        (venv) $ csv-reconcile serve                                      # run the service\n        (venv) $ deactivate                                               # remove virtual environment\n\nThe service is run at <http://127.0.0.1:5000/reconcile>. You can point\nat a different host:port by adding\n[SERVER\\_NAME](https://flask.palletsprojects.com/en/0.12.x/config/) to\nthe sample.cfg. Since this is running from a virtualenv, you can simply\ndelete the whole lot to clean up.\n\nIf you have a C compiler installed you may prefer to install the sdist\n`dist/csv-reconcile-0.1.0.tar.gz` which will build a\n[Cython](https://cython.readthedocs.io/en/latest/) version of the\ncomputationally intensive fuzzy match routine for speed. With `pip` add\nthe option `--no-binary csv-reconcile`.\n\n## Poetry\n\n### Prerequesites\n\nYou\'ll need to have both [poetry](https://python-poetry.org/docs/) and\n[poethepoet](https://pypi.org/project/poethepoet/0.0.3/) installed. For\npublishing to [PyPI](https://pypi.org/) [pandoc](https://pandoc.org/) is\nrequired.\n\n### Running\n\nThis is packaged with [poetry](https://python-poetry.org/docs/), so you\ncan use those commands if you have it installed.\n\n    $ poe install\n    $ poetry run csv-reconcile init sample/reps.tsv item itemLabel\n    $ poetry run csv-reconcile serve\n\n### Building\n\nBecause this package uses a `README.org` file and `pip` requires a\n`README.md`, there are extra build steps beyond what `poetry` supplies.\nThese are managed using\n[poethepoet](https://pypi.org/project/poethepoet/0.0.3/). Thus building\nis done as follows:\n\n    $ poe build\n\nIf you want to build a platform agnostic wheel, you\'ll have to comment\nout the `build =\n    "build.py"` line from `pyproject.toml` until `poetry` supports\n[selecting build\nplatform](https://github.com/python-poetry/poetry/issues/3594).\n\n## Description\n\nThis reconciliation service uses [Dice coefficient\nscoring](https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient)\nto reconcile values against a given column in a\n[CSV](https://en.wikipedia.org/wiki/Comma-separated_values) file. The\nCSV file must contain a column containing distinct values to reconcile\nto. We\'ll call this the *id column*. We\'ll call the column being\nreconciled against the *name column*.\n\nFor performance reasons, the *name column* is preprocessed to normalized\nvalues which are stored in an\n[sqlite](https://www.sqlite.org/index.html) database. This database must\nbe initialized at least once by running the init sub-command. Once\ninitialized this need not be run for subsequent runs.\n\nNote that the service supplies all its data with a dummy *type* so there\nis no reason to reconcile against any particular *type*.\n\nIn addition to reconciling against the *name column*, the service also\nfunctions as a [data extension\nservice](https://reconciliation-api.github.io/specs/latest/#data-extension-service),\nwhich offers any of the other columns of the CSV file.\n\nNote that Dice coefficient scoring is agnostic to word ordering.\n\n## Usage\n\nBasic usage involves two steps:\n\n-   initialization\n-   running the service\n\nInitialization primes the database with the data processed from the CSV\nfile with the `init` subcommand. There are several options for running\nthe service as described below.\n\n### Initialization\n\nBasic usage of the `init` sub-command requires passing the name of the\nCSV file, the *id column* and the *name column*.\n\n    (venv) $ csv-reconcile --help\n    Usage: csv-reconcile [OPTIONS] COMMAND [ARGS]...\n\n    Options:\n      --help  Show this message and exit.\n\n    Commands:\n      init\n      run\n      serve\n    (venv) $ csv-reconcile init --help\n    Usage: csv-reconcile init [OPTIONS] CSVFILE IDCOL NAMECOL\n\n    Options:\n      --config TEXT  config file\n      --scorer TEXT  scoring plugin to use\n      --help         Show this message and exit.\n    (venv) $ poetry run csv-reconcile serve --help\n    Usage: csv-reconcile serve [OPTIONS]\n\n    Options:\n      --help         Show this message and exit.\n    (venv) $\n\nThe `--config` option is used to point to a configuration file. The file\nis a [Flask\nconfiguration](https://flask.palletsprojects.com/en/1.1.x/config/) and\nhence is Python code though most configuration is simply setting\nvariables to constant values.\n\n### Running the service\n\nThe simplest way to run the service is to use Flask\'s built-in web\nserver with the `serve` subcommand which takes no arguments. However, as\nmentioned in the [Flask\ndocumentation](https://flask.palletsprojects.com/en/2.0.x/deploying/),\nthis server is not suitable for production purposes.\n\nFor a more hardened service, you can use one of the other deployment\noptions mentioned in that documentation. For example, gunicorn can be\nrun as follows:\n\n    (venv) $ gunicorn -w 4 \'csv_reconcile:create_app()\'\n    1-11-16 17:40:20 +0900] [84625] [INFO] Starting gunicorn 20.1.0\n    1-11-16 17:40:20 +0900] [84625] [INFO] Listening at: http://127.0.0.1:8000 (84625)\n    1-11-16 17:40:20 +0900] [84625] [INFO] Using worker: sync\n    1-11-16 17:40:20 +0900] [84626] [INFO] Booting worker with pid: 84626\n    1-11-16 17:40:20 +0900] [84627] [INFO] Booting worker with pid: 84627\n    1-11-16 17:40:20 +0900] [84628] [INFO] Booting worker with pid: 84628\n    1-11-16 17:40:20 +0900] [84629] [INFO] Booting worker with pid: 84629\n    ...\n\nOne thing to watch out for is that the default manifest points the\nextension service to port 5000, the default port for the Flask built-in\nweb server. If you want to use the extension service when deploying to a\ndifferent port, you\'ll want to be sure to override that part of the\nmanifest in your config file. You\'ll need something like the following:\n\n    MANIFEST = {\n        "extend": {\n            "propose_properties": {\n                "service_url": "http://localhost:8000",\n                "service_path": "/properties"\n            }\n        }\n    }\n\nNote also that the configuration is saved during the `init` step. If you\nchange the config, you\'ll need to re-run that step. You may also need to\ndelete and re-add the service in OpenRefine.\n\n### Deprecated\n\nThe `run` subcommand mimics the old behavior which combined the\ninitialization step with the running of the service. This may be removed\nin a future release.\n\n## Common configuration\n\n-   `SERVER_NAME` - The host and port the service is bound to. e.g.\n    `SERVER_NAME=localhost:5555`. ( Default localhost:5000 )\n-   `CSVKWARGS` - Arguments to pass to\n    [csv.reader](https://docs.python.org/3/library/csv.html). e.g.\n    `CSVKWARGS={\'delimiter\': \',\', \'quotechar\': \'"\'}` for comma delimited\n    files using `"` as quote character.\n-   `CSVENCODING` - Encoding of the CSV file. e.g.\n    `CSVENCODING="utf-8-sig"` is the encoding used for data downloaded\n    from\n    [GNIS](https://www.usgs.gov/core-science-systems/ngp/board-on-geographic-names/download-gnis-data).\n-   `SCOREOPTIONS` - Options passed to scoring plugin during\n    normalization. e.g.\n    `SCOREOPTIONS={\'stopwords\':[\'lake\',\'reservoir\']}`\n-   `LIMIT` - The maximum number of reonciliation candidates returned\n    per entry. ( Default 10 ) e.g. `LIMIT=10`\n-   `THRESHOLD` - The minimum score for returned reconciliation\n    candidates. ( Default 30.0 ) e.g. `THRESHOLD=80.5`\n-   `DATABASE` - The name of the generated sqlite database containing\n    pre-processed values. (Default `csvreconcile.db`) e.g.\n    `DATABASE=\'lakes.db\'` You may want to change the name of the\n    database if you regularly switch between databases being used.\n-   `MANIFEST` - Overrides for the service manifest. e.g.\n    `MANIFEST={"name": "My service"}` sets the name of the service to\n    "My service".\n\nThis last is most interesting. If your data is coming from\n[Wikidata](https://www.wikidata.org) and your *id column* contains [Q\nvalues](https://www.wikidata.org/wiki/Help:Items), then a manifest like\nthe following will allow your links to be clickable inside OpenRefine.\n\n    MANIFEST = {\n      "identifierSpace": "http://www.wikidata.org/entity/",\n      "schemaSpace": "http://www.wikidata.org/prop/direct/",\n      "view": {"url":"https://www.wikidata.org/wiki/{{id}}"},\n      "name": "My reconciliation service"\n    }\n\nIf your CSV is made up of data taken from another [reconciliation\nservice](https://reconciliation-api.github.io/testbench/), you may\nsimiliarly copy parts of their manifest to make use of their features,\nsuch as the [preview\nservice](https://reconciliation-api.github.io/specs/latest/#preview-service).\nSee the reconciliation spec for details.\n\n## Built-in preview service\n\nThere is a preview service built into the tool. (Thanks\n[b2m](https://github.com/b2m)!) You can turn it on by adding the\nfollowing to your manifest:\n\n    "preview": {\n       "url": "http://localhost:5000/preview/{{id}}",\n       "width": 400,\n       "height": 300\n    }\n\nNote that if you reconcile against a service with a preview service\nenabled, a link to the service becomes part of the project. Thus if you\nbring the service down, your project will have hover over pop-ups to an\nunavailable service. One way around this is to copy the `recon.match.id`\nto a new column which can be re-reconciled to the column by id if you\nbring the service back up again whether or not you have preview service\nenabled. (Perhaps OpenRefine could be smarter about enabling this\npop-ups only when the service is active.)\n\n## Scoring plugins\n\nAs mentioned above the default scoring method is to use [Dice\ncoefficient\nscoring](https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient),\nbut this method can be overridden by implementing a\n`cvs_reconcile.scorers` plugin.\n\n### Implementing\n\nA plugin module may override any of the methods in the\n`csv_reconcile.scorers` module by simply implementing a method of the\nsame name with the decorator `@cvs_reconcile.scorer.register`.\n\nSee `csv_reconcile_dice` for how Dice coefficient scoring is\nimplemented.\n\nThe basic hooks are as follows:\n\n-   `normalizedWord(word, **scoreOptions)` preprocesses values to be\n    reconciled to produce a tuple used in fuzzy match scoring. The value\n    of `SCOREOPTIONS` in the configuration will be passed in to allow\n    configuration of this preprocessing. This hook is required.\n-   `normalizedRow(word, row, **scoreOptions)` preprocesses values to be\n    reconciled against to produce a tuple used in fuzzy match scoring.\n    Note that both the reconciled column and the entire row is available\n    for calculating the normalized value and that the column reconciled\n    against is required even when not used. The value of `SCOREOPTIONS`\n    in the configuration will be passed in to allow configuration of\n    this preprocessing. This defaults to calling\n    normalizeWord(word,\\*\\*scoreOptions).\n-   `getNormalizedFields()` returns a tuple of names for the columns\n    produced by `normalizeWord()`. The length of the return value from\n    both functions must match. This defaults to calling\n    normalizeWord(word,\\*\\*scoreOptions). This hook is required.\n-   `processScoreOptions(options)` is passed the value of `SCOREOPTIONS`\n    to allow it to be adjusted prior to being used. This can be used for\n    adding defaults and/or validating the configuration. This hook is\n    optional\n-   `scoreMatch(left, right, **scoreOptions)` gets passed two tuples as\n    returned by `normalizedWord()`. The `left` value is the value being\n    reconciled and the `right` value is the value being reconciled\n    against. The value of `SCOREOPTIONS` in the configuration will be\n    passed in to allow configuration of this preprocessing. Returning a\n    score of `None` will not add tested value as a candidate. This hook\n    is required.\n-   `valid(normalizedFields)` is passed the normalized tuple prior to\n    being scored to make sure it\'s appropriate for the calculation. This\n    hook is optional.\n-   `features(word, row, **scoreOptions)` calculates\n    [features](https://reconciliation-api.github.io/specs/latest/#reconciliation-query-responses)\n    using the query string and the normalized row. By default\n    calculating features is disabled. Implementions of this hook are\n    automatically enabled. This hook is optional.\n\n### Installing\n\nHooks are automatically discovered as long as they provide a\n`csv_reconcile.scorers` [setuptools entry\npoint](https://setuptools.readthedocs.io/en/latest/userguide/entry_point.html).\nPoetry supplies a\n[plugins](https://python-poetry.org/docs/pyproject/#plugins)\nconfiguration which wraps the setuptools funtionality.\n\nThe default Dice coefficent scoring is supplied via the following\nsnippet from `pyproject.toml` file.\n\n    [tool.poetry.plugins."csv_reconcile.scorers"]\n    "dice" = "csv_reconcile_dice"\n\nHere `dice` becomes the name of the scoring option and\n`csv_reconcile_dice` is the package implementing the plugin.\n\n### Using\n\nIf there is only one scoring plugin available, that plugin is used. If\nthere are more than one available, you will be prompted to pass the\n`--scorer` option to select among the scoring options.\n\n### Known plugins\n\nSee [wiki](https://github.com/gitonthescene/csv-reconcile/wiki) for list\nof known plugins.\n\n## Testing\n\nThough I long for the old days when a unit test was a unit test, these\ndays things are a bit more complicated with various versions of `Python`\nand installation of plugins to manage. Now we have to wrestle with\n[virtual environments](https://docs.python.org/3/tutorial/venv.html).\n`poetry` handles the virtual environment for developing, but testing\ninvolves covering more options.\n\n### Tests layout\n\nThe tests directory structure is the following:\n\n    tests\n        main\n        plugins\n            geo\n\nTests for the main package are found under `main` and don\'t require\ninstalling any other packages whereas tests under `plugins` require the\ninstallation of the given plugin.\n\n### Running tests\n\n1.  Basic tests\n\n    These tests are written with\n    [pytest](https://docs.pytest.org/en/6.2.x/contents.html) and can be\n    running through `poetry` as follows:\n\n        $ poetry run pytest\n\n    To avoid the complications that come from installing plugins, there\n    is a `poe` script for running only the tests under main which can be\n    invoked as follows:\n\n        $ poe test\n\n    For steady state developing this is probably the command you\'ll use\n    most often.\n\n2.  Build matrices\n\n    The GitHub Actions for this project currently use a [build\n    matrix](https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix)\n    across a couple of architectures and several versions of `Python`,\n    but a similar effect can be achieved using\n    [nox](https://nox.thea.codes/en/stable/tutorial.html).\n\n    `nox` manages the creation of various virtual environments in what\n    they call "sessions", from which various commands can be run. This\n    project\'s `noxfile.py` manages the installation of the\n    `csv-reconcile-geo` plugin for the plugin tests as well as running\n    across several versions of `Python`. See the `nox` documentation for\n    detail.\n\n    Some versions of this command you\'re likely to run are as follows:\n\n        $ nox      # Run all the tests building virtual environemnts from scratch\n        $ nox -r   # Reuse previously built virtual environments for speed\n        $ nox -s test_geo  # Run only the tests for the csv-reconcile-geo plugin\n        $ nox -s test_main -p 3.8   # Run only the main tests with Python3.8\n\n    Eventually, the GitHub Actions may be changed to use\n    [setup-nox](https://github.com/marketplace/actions/setup-nox).\n\n## Future enhancements\n\nIt would be nice to add support for using\n[properties](https://reconciliation-api.github.io/specs/latest/#structure-of-a-reconciliation-query)\nas part of the scoring, so that more than one column of the csv could be\ntaken into consideration.\n',
    'author': 'Douglas Mennella',
    'author_email': 'trx2358-pypi@yahoo.com',
    'maintainer': None,
    'maintainer_email': None,
    'url': 'https://github.com/gitonthescene/csv-reconcile',
    'packages': packages,
    'package_data': package_data,
    'install_requires': install_requires,
    'extras_require': extras_require,
    'entry_points': entry_points,
    'python_requires': '>=3.7,<4.0',
}
from build import *
build(setup_kwargs)

setup(**setup_kwargs)
