Metadata-Version: 2.4
Name: ion-tokenizer
Version: 1.0.2
Summary: Phrase-first semantic tokenizer compiler for neural language models
Requires-Python: <3.14,>=3.9
License-File: LICENSE.md
Requires-Dist: click>=8.0
Requires-Dist: datasets>=2.14
Requires-Dist: spacy>=3.5
Requires-Dist: tqdm>=4.65
Requires-Dist: orjson>=3.9
Provides-Extra: embeddings
Requires-Dist: gensim>=4.3; extra == "embeddings"
Dynamic: license-file
