#AUTOGENERATED! DO NOT EDIT! File to edit: dev/02_nlp.ipynb (unless otherwise specified).

__all__ = ['nlp', 'tokenize', 'lexical_density']

#Cell
import os
import spacy

#Cell
nlp = spacy.load('en_core_web_sm')

#Cell
def tokenize(raw, lower=True):
    """Tokenize a string of text.

    Parameters
    -----------
    raw: str
        The text to tokenize.
    lower: bool
        If True, lowercase the input text.

    Returns
    --------
    list[str]
    """
    if lower: raw = raw.lower()
    return [t.text for t in nlp(raw, disable=['tagger', 'parser', 'ner'])]

#Cell
def lexical_density(text, lower=True):
    """Compute lexical density of a piece of text.

    https://en.wikipedia.org/wiki/Lexical_density

    Parameters
    -----------
    text: str or list[str]
    lower: bool (see `tokenize` docs)

    Returns
    --------
    float: Number between 0 and 1, where larger values indicate a higher
        lexical density.
    """
    if isinstance(text, str):
        text = tokenize(text, lower)
    if not text:
        return 0.0
    return len(set(text)) / len(text)