# -*- coding: utf-8 -*-
"""4_tpkenization split,SSpacy

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/12lglJVmIW-G98meeQv9I6zcSkNcKVD94

a. Tokenization using Python’s split() function
"""

text = """ This tool is an a beta stage. Alexa developers can use Get Metrics API to
seamlessly analyse metric. It also supports custom skill model, prebuilt Flash Briefing
model, and the Smart Home Skill API. You can use this tool for creation of monitors,
alarms, and dashboards that spotlight changes. The release of these three tools will
enable developers to create visual rich skills for Alexa devices with screens. Amazon
describes these tools as the collection of tech and tools for creating visually rich and
interactive voice experiences. """
data = text.split('.')
for i in data:
 print (i)

"""b. Tokenization using Regular Expressions (RegEx)

"""

import nltk
from nltk.tokenize import RegexpTokenizer
tk = RegexpTokenizer(r'\s+', gaps = True)
str = "I love to study Natural Language Processing in Python"
tokens = tk.tokenize(str)
print(tokens)

"""c. Tokenization using NLTK

"""

import nltk
nltk.download('punkt_tab')
from nltk.tokenize import word_tokenize
str = "I love to study Natural Language Processing in Python"
print(word_tokenize(str))

"""d. Tokenization using the spaCy library

"""

import spacy
nlp = spacy.blank("en")
str = "I love to study Natural Language Processing in Python"
doc = nlp(str)
words = [word.text for word in doc]
print(words)








print("second code ")




text = """This tool is an a beta stage..."""

# a. split()
data = text.split('.')
for i in data:
    print(i)

# b. RegEx
import nltk
from nltk.tokenize import RegexpTokenizer

tk = RegexpTokenizer(r'\s+', gaps=True)
text_str = "I love to study Natural Language Processing in Python"
tokens = tk.tokenize(text_str)
print(tokens)

# c. NLTK
nltk.download('punkt')
from nltk.tokenize import word_tokenize
print(word_tokenize(text_str))

# d. spaCy
import spacy
nlp = spacy.blank("en")
doc = nlp(text_str)
words = [word.text for word in doc]
print(words)

