import pytest
from pylexitext.text import Text

text_sample = "I got a lot of questions about programming but i don't think they're the right questions to ask questions like. How many languages do i need to learn what language do i need to learn to get to company x what's the difference between a and b and which makes more money what are the exact steps i need to take. To become the. What kind of program should i write to build up my resume so why don't i like these questions because it feels like the people who are asking these questions they are learning how to program just for the sake of learning how to program. The program is just a tool. Being a programmer for me means you write code to automate things. Or to make things perform certain actions for example programming a refrigerator to turn on the light when you open the door. When you think about it. Programming is actually kind of boring. You're just giving a series of instructions to a machine over and over again until you get it right i don't want to speak for all of you. But i think it's a pay wasn't great i don't think a lot of you would spend hours and hours every night in front of the computer trying to learn how to code or to just debug your program sometimes we get caught up learning languages or doing coding challenges or fantasizing about you not working for these big tech companies that weforget why we wanted to learn programming in the first place. You wanted to program because you saw the amazing things that can be built using programming. So don't just learn how to program build something that matters build something that means something to you build something that solves a problem you're a problem solver a creator an innovator you're not a programmer programming is just a tool in your arsenal to build something amazing. You can be analytical trade of empathetic and programming allows you to express those qualities of yourself but programming in of itself is nothing special it's like a pencil it can mark stuff down if you press on the pencil hard enough. That's it but with a pencil you can write novels draw beautiful portraits build plans for a skyscraper or anything limited only by your imagination. You don't learn how to program to get into google. You learn how to program to build something meaningful something that helps real people with real problems i'm partnered of ibm today to talk about a global initiative called call for code which calls for developers to build something impactful and have a positive change across the world through their code as you know there are hundreds and hundreds of natural disasters every year like hurricanes and earthquakes floods volcanoes and wildfires those affect numerous lies and causes tremendous damage to many families around the world if you strive for real impact we desperately need you this is why 2018 call for code global challenge is a competition that ass people like you to build solutions to improve what we can do to reduce the destructive impact of his natural disasters we need all kinds of technologies."


@pytest.fixture
def text():
    return Text(text_sample)


def test_health_check(text):
    assert text is not None

    try:
        x = Text(312312312312312312312312312312312312312)
        assert False
    except:
        assert True

    try:
        x = Text(text_sample, language='japanese')
        assert False
    except:
        assert True


def test_describe(text):
    features = text.describe()
    assert isinstance(features, dict)
    assert len(features.keys()) >= 18


def ngrams_extraction(text):
    ngrams_sample = [['i', 'got', 'a', 'lot'], ['got', 'a', 'lot', 'of'], ['a', 'lot', 'of', 'questions'], ['lot', 'of', 'questions', 'about'], ['of', 'questions', 'about', 'programming'], ['questions', 'about', 'programming', 'but'], ['about', 'programming', 'but', 'i'], ['programming', 'but', 'i', "don't"], ['but', 'i', "don't", 'think'], ['i', "don't", 'think', "they're"], ["don't", 'think', "they're", 'the'], ['think', "they're", 'the', 'right'], ["they're", 'the', 'right', 'questions'], ['the', 'right', 'questions', 'to'], ['right', 'questions', 'to', 'ask'], ['questions', 'to', 'ask', 'questions'], ['to', 'ask', 'questions', 'like.'], ['ask', 'questions', 'like.', 'how'], ['questions', 'like.', 'how', 'many'], ['like.', 'how', 'many', 'languages'], ['how', 'many', 'languages', 'do'], ['many', 'languages', 'do', 'i'], ['languages', 'do', 'i', 'need'], ['do', 'i', 'need', 'to'], ['i', 'need', 'to', 'learn'], ['need', 'to', 'learn', 'what'], ['to', 'learn', 'what', 'language'], ['learn', 'what', 'language', 'do'], ['what', 'language', 'do', 'i'], ['language', 'do', 'i', 'need'], ['do', 'i', 'need', 'to'], ['i', 'need', 'to', 'learn'], ['need', 'to', 'learn', 'to'], ['to', 'learn', 'to', 'get'], ['learn', 'to', 'get', 'to'], ['to', 'get', 'to', 'company'], ['get', 'to', 'company', 'x'], ['to', 'company', 'x', "what's"], ['company', 'x', "what's", 'the'], ['x', "what's", 'the', 'difference'], ["what's", 'the', 'difference', 'between'], ['the', 'difference', 'between', 'a'], ['difference', 'between', 'a', 'and'], ['between', 'a', 'and', 'b'], ['a', 'and', 'b', 'and'], ['and', 'b', 'and', 'which'], ['b', 'and', 'which', 'makes'], ['and', 'which', 'makes', 'more'], ['which', 'makes', 'more', 'money'], ['makes', 'more', 'money', 'what'], ['more', 'money', 'what', 'are'], ['money', 'what', 'are', 'the'], ['what', 'are', 'the', 'exact'], ['are', 'the', 'exact', 'steps'], ['the', 'exact', 'steps', 'i'], ['exact', 'steps', 'i', 'need'], ['steps', 'i', 'need', 'to'], ['i', 'need', 'to', 'take.'], ['need', 'to', 'take.', 'to'], ['to', 'take.', 'to', 'become'], ['take.', 'to', 'become', 'the.'], ['to', 'become', 'the.', 'what'], ['become', 'the.', 'what', 'kind'], ['the.', 'what', 'kind', 'of'], ['what', 'kind', 'of', 'program'], ['kind', 'of', 'program', 'should'], ['of', 'program', 'should', 'i'], ['program', 'should', 'i', 'write'], ['should', 'i', 'write', 'to'], ['i', 'write', 'to', 'build'], ['write', 'to', 'build', 'up'], ['to', 'build', 'up', 'my'], ['build', 'up', 'my', 'resume'], ['up', 'my', 'resume', 'so'], ['my', 'resume', 'so', 'why'], ['resume', 'so', 'why', "don't"], ['so', 'why', "don't", 'i'], ['why', "don't", 'i', 'like'], ["don't", 'i', 'like', 'these'], ['i', 'like', 'these', 'questions'], ['like', 'these', 'questions', 'because'], ['these', 'questions', 'because', 'it'], ['questions', 'because', 'it', 'feels'], ['because', 'it', 'feels', 'like'], ['it', 'feels', 'like', 'the'], ['feels', 'like', 'the', 'people'], ['like', 'the', 'people', 'who'], ['the', 'people', 'who', 'are'], ['people', 'who', 'are', 'asking'], ['who', 'are', 'asking', 'these'], ['are', 'asking', 'these', 'questions'], ['asking', 'these', 'questions', 'they'], ['these', 'questions', 'they', 'are'], ['questions', 'they', 'are', 'learning'], ['they', 'are', 'learning', 'how'], ['are', 'learning', 'how', 'to'], ['learning', 'how', 'to', 'program'], ['how', 'to', 'program', 'just'], ['to', 'program', 'just', 'for'], ['program', 'just', 'for', 'the'], ['just', 'for', 'the', 'sake'], ['for', 'the', 'sake', 'of'], ['the', 'sake', 'of', 'learning'], ['sake', 'of', 'learning', 'how'], ['of', 'learning', 'how', 'to'], ['learning', 'how', 'to', 'program.'], ['how', 'to', 'program.', 'the'], ['to', 'program.', 'the', 'program'], ['program.', 'the', 'program', 'is'], ['the', 'program', 'is', 'just'], ['program', 'is', 'just', 'a'], ['is', 'just', 'a', 'tool.'], ['just', 'a', 'tool.', 'being'], ['a', 'tool.', 'being', 'a'], ['tool.', 'being', 'a', 'programmer'], ['being', 'a', 'programmer', 'for'], ['a', 'programmer', 'for', 'me'], ['programmer', 'for', 'me', 'means'], ['for', 'me', 'means', 'you'], ['me', 'means', 'you', 'write'], ['means', 'you', 'write', 'code'], ['you', 'write', 'code', 'to'], ['write', 'code', 'to', 'automate'], ['code', 'to', 'automate', 'things.'], ['to', 'automate', 'things.', 'or'], ['automate', 'things.', 'or', 'to'], ['things.', 'or', 'to', 'make'], ['or', 'to', 'make', 'things'], ['to', 'make', 'things', 'perform'], ['make', 'things', 'perform', 'certain'], ['things', 'perform', 'certain', 'actions'], ['perform', 'certain', 'actions', 'for'], ['certain', 'actions', 'for', 'example'], ['actions', 'for', 'example', 'programming'], ['for', 'example', 'programming', 'a'], ['example', 'programming', 'a', 'refrigerator'], ['programming', 'a', 'refrigerator', 'to'], ['a', 'refrigerator', 'to', 'turn'], ['refrigerator', 'to', 'turn', 'on'], ['to', 'turn', 'on', 'the'], ['turn', 'on', 'the', 'light'], ['on', 'the', 'light', 'when'], ['the', 'light', 'when', 'you'], ['light', 'when', 'you', 'open'], ['when', 'you', 'open', 'the'], ['you', 'open', 'the', 'door.'], ['open', 'the', 'door.', 'when'], ['the', 'door.', 'when', 'you'], ['door.', 'when', 'you', 'think'], ['when', 'you', 'think', 'about'], ['you', 'think', 'about', 'it.'], ['think', 'about', 'it.', 'programming'], ['about', 'it.', 'programming', 'is'], ['it.', 'programming', 'is', 'actually'], ['programming', 'is', 'actually', 'kind'], ['is', 'actually', 'kind', 'of'], ['actually', 'kind', 'of', 'boring.'], ['kind', 'of', 'boring.', "you're"], ['of', 'boring.', "you're", 'just'], ['boring.', "you're", 'just', 'giving'], ["you're", 'just', 'giving', 'a'], ['just', 'giving', 'a', 'series'], ['giving', 'a', 'series', 'of'], ['a', 'series', 'of', 'instructions'], ['series', 'of', 'instructions', 'to'], ['of', 'instructions', 'to', 'a'], ['instructions', 'to', 'a', 'machine'], ['to', 'a', 'machine', 'over'], ['a', 'machine', 'over', 'and'], ['machine', 'over', 'and', 'over'], ['over', 'and', 'over', 'again'], ['and', 'over', 'again', 'until'], ['over', 'again', 'until', 'you'], ['again', 'until', 'you', 'get'], ['until', 'you', 'get', 'it'], ['you', 'get', 'it', 'right'], ['get', 'it', 'right', 'i'], ['it', 'right', 'i', "don't"], ['right', 'i', "don't", 'want'], ['i', "don't", 'want', 'to'], ["don't", 'want', 'to', 'speak'], ['want', 'to', 'speak', 'for'], ['to', 'speak', 'for', 'all'], ['speak', 'for', 'all', 'of'], ['for', 'all', 'of', 'you.'], ['all', 'of', 'you.', 'but'], ['of', 'you.', 'but', 'i'], ['you.', 'but', 'i', 'think'], ['but', 'i', 'think', "it's"], ['i', 'think', "it's", 'a'], ['think', "it's", 'a', 'pay'], ["it's", 'a', 'pay', "wasn't"], ['a', 'pay', "wasn't", 'great'], ['pay', "wasn't", 'great', 'i'], ["wasn't", 'great', 'i', "don't"], ['great', 'i', "don't", 'think'], ['i', "don't", 'think', 'a'], ["don't", 'think', 'a', 'lot'], ['think', 'a', 'lot', 'of'], ['a', 'lot', 'of', 'you'], ['lot', 'of', 'you', 'would'], ['of', 'you', 'would', 'spend'], ['you', 'would', 'spend', 'hours'], ['would', 'spend', 'hours', 'and'], ['spend', 'hours', 'and', 'hours'], ['hours', 'and', 'hours', 'every'], ['and', 'hours', 'every', 'night'], ['hours', 'every', 'night', 'in'], ['every', 'night', 'in', 'front'], ['night', 'in', 'front', 'of'], ['in', 'front', 'of', 'the'], ['front', 'of', 'the', 'computer'], ['of', 'the', 'computer', 'trying'], ['the', 'computer', 'trying', 'to'], ['computer', 'trying', 'to', 'learn'], ['trying', 'to', 'learn', 'how'], ['to', 'learn', 'how', 'to'], ['learn', 'how', 'to', 'code'], ['how', 'to', 'code', 'or'], ['to', 'code', 'or', 'to'], ['code', 'or', 'to', 'just'], ['or', 'to', 'just', 'debug'], ['to', 'just', 'debug', 'your'], ['just', 'debug', 'your', 'program'], ['debug', 'your', 'program', 'sometimes'], ['your', 'program', 'sometimes', 'we'], ['program', 'sometimes', 'we', 'get'], ['sometimes', 'we', 'get', 'caught'], ['we', 'get', 'caught', 'up'], ['get', 'caught', 'up', 'learning'], ['caught', 'up', 'learning', 'languages'], ['up', 'learning', 'languages', 'or'], ['learning', 'languages', 'or', 'doing'], ['languages', 'or', 'doing', 'coding'], ['or', 'doing', 'coding', 'challenges'], ['doing', 'coding', 'challenges', 'or'], ['coding', 'challenges', 'or', 'fantasizing'], ['challenges', 'or', 'fantasizing', 'about'], ['or', 'fantasizing', 'about', 'you'], ['fantasizing', 'about', 'you', 'not'], ['about', 'you', 'not', 'working'], ['you', 'not', 'working', 'for'], ['not', 'working', 'for', 'these'], ['working', 'for', 'these', 'big'], ['for', 'these', 'big', 'tech'], ['these', 'big', 'tech', 'companies'], ['big', 'tech', 'companies', 'that'], ['tech', 'companies', 'that', 'weforget'], ['companies', 'that', 'weforget', 'why'], ['that', 'weforget', 'why', 'we'], ['weforget', 'why', 'we', 'wanted'], ['why', 'we', 'wanted', 'to'], ['we', 'wanted', 'to', 'learn'], ['wanted', 'to', 'learn', 'programming'], ['to', 'learn', 'programming', 'in'], ['learn', 'programming', 'in', 'the'], ['programming', 'in', 'the', 'first'], ['in', 'the', 'first', 'place.'], ['the', 'first', 'place.', 'you'], ['first', 'place.', 'you', 'wanted'], ['place.', 'you', 'wanted', 'to'], ['you', 'wanted', 'to', 'program'], ['wanted', 'to', 'program', 'because'], ['to', 'program', 'because', 'you'], ['program', 'because', 'you', 'saw'], ['because', 'you', 'saw', 'the'], ['you', 'saw', 'the', 'amazing'], ['saw', 'the', 'amazing', 'things'], ['the', 'amazing', 'things', 'that'], ['amazing', 'things', 'that', 'can'], ['things', 'that', 'can', 'be'], ['that', 'can', 'be', 'built'], ['can', 'be', 'built', 'using'], ['be', 'built', 'using', 'programming.'], ['built', 'using', 'programming.', 'so'], ['using', 'programming.', 'so', "don't"], [
        'programming.', 'so', "don't", 'just'], ['so', "don't", 'just', 'learn'], ["don't", 'just', 'learn', 'how'], ['just', 'learn', 'how', 'to'], ['learn', 'how', 'to', 'program'], ['how', 'to', 'program', 'build'], ['to', 'program', 'build', 'something'], ['program', 'build', 'something', 'that'], ['build', 'something', 'that', 'matters'], ['something', 'that', 'matters', 'build'], ['that', 'matters', 'build', 'something'], ['matters', 'build', 'something', 'that'], ['build', 'something', 'that', 'means'], ['something', 'that', 'means', 'something'], ['that', 'means', 'something', 'to'], ['means', 'something', 'to', 'you'], ['something', 'to', 'you', 'build'], ['to', 'you', 'build', 'something'], ['you', 'build', 'something', 'that'], ['build', 'something', 'that', 'solves'], ['something', 'that', 'solves', 'a'], ['that', 'solves', 'a', 'problem'], ['solves', 'a', 'problem', "you're"], ['a', 'problem', "you're", 'a'], ['problem', "you're", 'a', 'problem'], ["you're", 'a', 'problem', 'solver'], ['a', 'problem', 'solver', 'a'], ['problem', 'solver', 'a', 'creator'], ['solver', 'a', 'creator', 'an'], ['a', 'creator', 'an', 'innovator'], ['creator', 'an', 'innovator', "you're"], ['an', 'innovator', "you're", 'not'], ['innovator', "you're", 'not', 'a'], ["you're", 'not', 'a', 'programmer'], ['not', 'a', 'programmer', 'programming'], ['a', 'programmer', 'programming', 'is'], ['programmer', 'programming', 'is', 'just'], ['programming', 'is', 'just', 'a'], ['is', 'just', 'a', 'tool'], ['just', 'a', 'tool', 'in'], ['a', 'tool', 'in', 'your'], ['tool', 'in', 'your', 'arsenal'], ['in', 'your', 'arsenal', 'to'], ['your', 'arsenal', 'to', 'build'], ['arsenal', 'to', 'build', 'something'], ['to', 'build', 'something', 'amazing.'], ['build', 'something', 'amazing.', 'you'], ['something', 'amazing.', 'you', 'can'], ['amazing.', 'you', 'can', 'be'], ['you', 'can', 'be', 'analytical'], ['can', 'be', 'analytical', 'trade'], ['be', 'analytical', 'trade', 'of'], ['analytical', 'trade', 'of', 'empathetic'], ['trade', 'of', 'empathetic', 'and'], ['of', 'empathetic', 'and', 'programming'], ['empathetic', 'and', 'programming', 'allows'], ['and', 'programming', 'allows', 'you'], ['programming', 'allows', 'you', 'to'], ['allows', 'you', 'to', 'express'], ['you', 'to', 'express', 'those'], ['to', 'express', 'those', 'qualities'], ['express', 'those', 'qualities', 'of'], ['those', 'qualities', 'of', 'yourself'], ['qualities', 'of', 'yourself', 'but'], ['of', 'yourself', 'but', 'programming'], ['yourself', 'but', 'programming', 'in'], ['but', 'programming', 'in', 'of'], ['programming', 'in', 'of', 'itself'], ['in', 'of', 'itself', 'is'], ['of', 'itself', 'is', 'nothing'], ['itself', 'is', 'nothing', 'special'], ['is', 'nothing', 'special', "it's"], ['nothing', 'special', "it's", 'like'], ['special', "it's", 'like', 'a'], ["it's", 'like', 'a', 'pencil'], ['like', 'a', 'pencil', 'it'], ['a', 'pencil', 'it', 'can'], ['pencil', 'it', 'can', 'mark'], ['it', 'can', 'mark', 'stuff'], ['can', 'mark', 'stuff', 'down'], ['mark', 'stuff', 'down', 'if'], ['stuff', 'down', 'if', 'you'], ['down', 'if', 'you', 'press'], ['if', 'you', 'press', 'on'], ['you', 'press', 'on', 'the'], ['press', 'on', 'the', 'pencil'], ['on', 'the', 'pencil', 'hard'], ['the', 'pencil', 'hard', 'enough.'], ['pencil', 'hard', 'enough.', "that's"], ['hard', 'enough.', "that's", 'it'], ['enough.', "that's", 'it', 'but'], ["that's", 'it', 'but', 'with'], ['it', 'but', 'with', 'a'], ['but', 'with', 'a', 'pencil'], ['with', 'a', 'pencil', 'you'], ['a', 'pencil', 'you', 'can'], ['pencil', 'you', 'can', 'write'], ['you', 'can', 'write', 'novels'], ['can', 'write', 'novels', 'draw'], ['write', 'novels', 'draw', 'beautiful'], ['novels', 'draw', 'beautiful', 'portraits'], ['draw', 'beautiful', 'portraits', 'build'], ['beautiful', 'portraits', 'build', 'plans'], ['portraits', 'build', 'plans', 'for'], ['build', 'plans', 'for', 'a'], ['plans', 'for', 'a', 'skyscraper'], ['for', 'a', 'skyscraper', 'or'], ['a', 'skyscraper', 'or', 'anything'], ['skyscraper', 'or', 'anything', 'limited'], ['or', 'anything', 'limited', 'only'], ['anything', 'limited', 'only', 'by'], ['limited', 'only', 'by', 'your'], ['only', 'by', 'your', 'imagination.'], ['by', 'your', 'imagination.', 'you'], ['your', 'imagination.', 'you', "don't"], ['imagination.', 'you', "don't", 'learn'], ['you', "don't", 'learn', 'how'], ["don't", 'learn', 'how', 'to'], ['learn', 'how', 'to', 'program'], ['how', 'to', 'program', 'to'], ['to', 'program', 'to', 'get'], ['program', 'to', 'get', 'into'], ['to', 'get', 'into', 'google.'], ['get', 'into', 'google.', 'you'], ['into', 'google.', 'you', 'learn'], ['google.', 'you', 'learn', 'how'], ['you', 'learn', 'how', 'to'], ['learn', 'how', 'to', 'program'], ['how', 'to', 'program', 'to'], ['to', 'program', 'to', 'build'], ['program', 'to', 'build', 'something'], ['to', 'build', 'something', 'meaningful'], ['build', 'something', 'meaningful', 'something'], ['something', 'meaningful', 'something', 'that'], ['meaningful', 'something', 'that', 'helps'], ['something', 'that', 'helps', 'real'], ['that', 'helps', 'real', 'people'], ['helps', 'real', 'people', 'with'], ['real', 'people', 'with', 'real'], ['people', 'with', 'real', 'problems'], ['with', 'real', 'problems', "i'm"], ['real', 'problems', "i'm", 'partnered'], ['problems', "i'm", 'partnered', 'of'], ["i'm", 'partnered', 'of', 'ibm'], ['partnered', 'of', 'ibm', 'today'], ['of', 'ibm', 'today', 'to'], ['ibm', 'today', 'to', 'talk'], ['today', 'to', 'talk', 'about'], ['to', 'talk', 'about', 'a'], ['talk', 'about', 'a', 'global'], ['about', 'a', 'global', 'initiative'], ['a', 'global', 'initiative', 'called'], ['global', 'initiative', 'called', 'call'], ['initiative', 'called', 'call', 'for'], ['called', 'call', 'for', 'code'], ['call', 'for', 'code', 'which'], ['for', 'code', 'which', 'calls'], ['code', 'which', 'calls', 'for'], ['which', 'calls', 'for', 'developers'], ['calls', 'for', 'developers', 'to'], ['for', 'developers', 'to', 'build'], ['developers', 'to', 'build', 'something'], ['to', 'build', 'something', 'impactful'], ['build', 'something', 'impactful', 'and'], ['something', 'impactful', 'and', 'have'], ['impactful', 'and', 'have', 'a'], ['and', 'have', 'a', 'positive'], ['have', 'a', 'positive', 'change'], ['a', 'positive', 'change', 'across'], ['positive', 'change', 'across', 'the'], ['change', 'across', 'the', 'world'], ['across', 'the', 'world', 'through'], ['the', 'world', 'through', 'their'], ['world', 'through', 'their', 'code'], ['through', 'their', 'code', 'as'], ['their', 'code', 'as', 'you'], ['code', 'as', 'you', 'know'], ['as', 'you', 'know', 'there'], ['you', 'know', 'there', 'are'], ['know', 'there', 'are', 'hundreds'], ['there', 'are', 'hundreds', 'and'], ['are', 'hundreds', 'and', 'hundreds'], ['hundreds', 'and', 'hundreds', 'of'], ['and', 'hundreds', 'of', 'natural'], ['hundreds', 'of', 'natural', 'disasters'], ['of', 'natural', 'disasters', 'every'], ['natural', 'disasters', 'every', 'year'], ['disasters', 'every', 'year', 'like'], ['every', 'year', 'like', 'hurricanes'], ['year', 'like', 'hurricanes', 'and'], ['like', 'hurricanes', 'and', 'earthquakes'], ['hurricanes', 'and', 'earthquakes', 'floods'], ['and', 'earthquakes', 'floods', 'volcanoes'], ['earthquakes', 'floods', 'volcanoes', 'and'], ['floods', 'volcanoes', 'and', 'wildfires'], ['volcanoes', 'and', 'wildfires', 'those'], ['and', 'wildfires', 'those', 'affect'], ['wildfires', 'those', 'affect', 'numerous'], ['those', 'affect', 'numerous', 'lies'], ['affect', 'numerous', 'lies', 'and'], ['numerous', 'lies', 'and', 'causes'], ['lies', 'and', 'causes', 'tremendous'], ['and', 'causes', 'tremendous', 'damage'], ['causes', 'tremendous', 'damage', 'to'], ['tremendous', 'damage', 'to', 'many'], ['damage', 'to', 'many', 'families'], ['to', 'many', 'families', 'around'], ['many', 'families', 'around', 'the'], ['families', 'around', 'the', 'world'], ['around', 'the', 'world', 'if'], ['the', 'world', 'if', 'you'], ['world', 'if', 'you', 'strive'], ['if', 'you', 'strive', 'for'], ['you', 'strive', 'for', 'real'], ['strive', 'for', 'real', 'impact'], ['for', 'real', 'impact', 'we'], ['real', 'impact', 'we', 'desperately'], ['impact', 'we', 'desperately', 'need'], ['we', 'desperately', 'need', 'you'], ['desperately', 'need', 'you', 'this'], ['need', 'you', 'this', 'is'], ['you', 'this', 'is', 'why'], ['this', 'is', 'why', '2018'], ['is', 'why', '2018', 'call'], ['why', '2018', 'call', 'for'], ['2018', 'call', 'for', 'code'], ['call', 'for', 'code', 'global'], ['for', 'code', 'global', 'challenge'], ['code', 'global', 'challenge', 'is'], ['global', 'challenge', 'is', 'a'], ['challenge', 'is', 'a', 'competition'], ['is', 'a', 'competition', 'that'], ['a', 'competition', 'that', 'ass'], ['competition', 'that', 'ass', 'people'], ['that', 'ass', 'people', 'like'], ['ass', 'people', 'like', 'you'], ['people', 'like', 'you', 'to'], ['like', 'you', 'to', 'build'], ['you', 'to', 'build', 'solutions'], ['to', 'build', 'solutions', 'to'], ['build', 'solutions', 'to', 'improve'], ['solutions', 'to', 'improve', 'what'], ['to', 'improve', 'what', 'we'], ['improve', 'what', 'we', 'can'], ['what', 'we', 'can', 'do'], ['we', 'can', 'do', 'to'], ['can', 'do', 'to', 'reduce'], ['do', 'to', 'reduce', 'the'], ['to', 'reduce', 'the', 'destructive'], ['reduce', 'the', 'destructive', 'impact'], ['the', 'destructive', 'impact', 'of'], ['destructive', 'impact', 'of', 'his'], ['impact', 'of', 'his', 'natural'], ['of', 'his', 'natural', 'disasters'], ['his', 'natural', 'disasters', 'we'], ['natural', 'disasters', 'we', 'need'], ['disasters', 'we', 'need', 'all'], ['we', 'need', 'all', 'kinds'], ['need', 'all', 'kinds', 'of'], ['all', 'kinds', 'of', 'technologies.']]
    assert text.ngrams_extraction(n=4) == ngrams_sample


def bigrams_extraction(text):
    bigrams = [['i', 'got'], ['got', 'a'], ['a', 'lot'], ['lot', 'of'], ['of', 'questions'], ['questions', 'about'], ['about', 'programming'], ['programming', 'but'], ['but', 'i'], ['i', "don't"], ["don't", 'think'], ['think', "they're"], ["they're", 'the'], ['the', 'right'], ['right', 'questions'], ['questions', 'to'], ['to', 'ask'], ['ask', 'questions'], ['questions', 'like.'], ['like.', 'how'], ['how', 'many'], ['many', 'languages'], ['languages', 'do'], ['do', 'i'], ['i', 'need'], ['need', 'to'], ['to', 'learn'], ['learn', 'what'], ['what', 'language'], ['language', 'do'], ['do', 'i'], ['i', 'need'], ['need', 'to'], ['to', 'learn'], ['learn', 'to'], ['to', 'get'], ['get', 'to'], ['to', 'company'], ['company', 'x'], ['x', "what's"], ["what's", 'the'], ['the', 'difference'], ['difference', 'between'], ['between', 'a'], ['a', 'and'], ['and', 'b'], ['b', 'and'], ['and', 'which'], ['which', 'makes'], ['makes', 'more'], ['more', 'money'], ['money', 'what'], ['what', 'are'], ['are', 'the'], ['the', 'exact'], ['exact', 'steps'], ['steps', 'i'], ['i', 'need'], ['need', 'to'], ['to', 'take.'], ['take.', 'to'], ['to', 'become'], ['become', 'the.'], ['the.', 'what'], ['what', 'kind'], ['kind', 'of'], ['of', 'program'], ['program', 'should'], ['should', 'i'], ['i', 'write'], ['write', 'to'], ['to', 'build'], ['build', 'up'], ['up', 'my'], ['my', 'resume'], ['resume', 'so'], ['so', 'why'], ['why', "don't"], ["don't", 'i'], ['i', 'like'], ['like', 'these'], ['these', 'questions'], ['questions', 'because'], ['because', 'it'], ['it', 'feels'], ['feels', 'like'], ['like', 'the'], ['the', 'people'], ['people', 'who'], ['who', 'are'], ['are', 'asking'], ['asking', 'these'], ['these', 'questions'], ['questions', 'they'], ['they', 'are'], ['are', 'learning'], ['learning', 'how'], ['how', 'to'], ['to', 'program'], ['program', 'just'], ['just', 'for'], ['for', 'the'], ['the', 'sake'], ['sake', 'of'], ['of', 'learning'], ['learning', 'how'], ['how', 'to'], ['to', 'program.'], ['program.', 'the'], ['the', 'program'], ['program', 'is'], ['is', 'just'], ['just', 'a'], ['a', 'tool.'], ['tool.', 'being'], ['being', 'a'], ['a', 'programmer'], ['programmer', 'for'], ['for', 'me'], ['me', 'means'], ['means', 'you'], ['you', 'write'], ['write', 'code'], ['code', 'to'], ['to', 'automate'], ['automate', 'things.'], ['things.', 'or'], ['or', 'to'], ['to', 'make'], ['make', 'things'], ['things', 'perform'], ['perform', 'certain'], ['certain', 'actions'], ['actions', 'for'], ['for', 'example'], ['example', 'programming'], ['programming', 'a'], ['a', 'refrigerator'], ['refrigerator', 'to'], ['to', 'turn'], ['turn', 'on'], ['on', 'the'], ['the', 'light'], ['light', 'when'], ['when', 'you'], ['you', 'open'], ['open', 'the'], ['the', 'door.'], ['door.', 'when'], ['when', 'you'], ['you', 'think'], ['think', 'about'], ['about', 'it.'], ['it.', 'programming'], ['programming', 'is'], ['is', 'actually'], ['actually', 'kind'], ['kind', 'of'], ['of', 'boring.'], ['boring.', "you're"], ["you're", 'just'], ['just', 'giving'], ['giving', 'a'], ['a', 'series'], ['series', 'of'], ['of', 'instructions'], ['instructions', 'to'], ['to', 'a'], ['a', 'machine'], ['machine', 'over'], ['over', 'and'], ['and', 'over'], ['over', 'again'], ['again', 'until'], ['until', 'you'], ['you', 'get'], ['get', 'it'], ['it', 'right'], ['right', 'i'], ['i', "don't"], ["don't", 'want'], ['want', 'to'], ['to', 'speak'], ['speak', 'for'], ['for', 'all'], ['all', 'of'], ['of', 'you.'], ['you.', 'but'], ['but', 'i'], ['i', 'think'], ['think', "it's"], ["it's", 'a'], ['a', 'pay'], ['pay', "wasn't"], ["wasn't", 'great'], ['great', 'i'], ['i', "don't"], ["don't", 'think'], ['think', 'a'], ['a', 'lot'], ['lot', 'of'], ['of', 'you'], ['you', 'would'], ['would', 'spend'], ['spend', 'hours'], ['hours', 'and'], ['and', 'hours'], ['hours', 'every'], ['every', 'night'], ['night', 'in'], ['in', 'front'], ['front', 'of'], ['of', 'the'], ['the', 'computer'], ['computer', 'trying'], ['trying', 'to'], ['to', 'learn'], ['learn', 'how'], ['how', 'to'], ['to', 'code'], ['code', 'or'], ['or', 'to'], ['to', 'just'], ['just', 'debug'], ['debug', 'your'], ['your', 'program'], ['program', 'sometimes'], ['sometimes', 'we'], ['we', 'get'], ['get', 'caught'], ['caught', 'up'], ['up', 'learning'], ['learning', 'languages'], ['languages', 'or'], ['or', 'doing'], ['doing', 'coding'], ['coding', 'challenges'], ['challenges', 'or'], ['or', 'fantasizing'], ['fantasizing', 'about'], ['about', 'you'], ['you', 'not'], ['not', 'working'], ['working', 'for'], ['for', 'these'], ['these', 'big'], ['big', 'tech'], ['tech', 'companies'], ['companies', 'that'], ['that', 'weforget'], ['weforget', 'why'], ['why', 'we'], ['we', 'wanted'], ['wanted', 'to'], ['to', 'learn'], ['learn', 'programming'], ['programming', 'in'], ['in', 'the'], ['the', 'first'], ['first', 'place.'], ['place.', 'you'], ['you', 'wanted'], ['wanted', 'to'], ['to', 'program'], ['program', 'because'], ['because', 'you'], ['you', 'saw'], ['saw', 'the'], ['the', 'amazing'], ['amazing', 'things'], ['things', 'that'], ['that', 'can'], ['can', 'be'], ['be', 'built'], ['built', 'using'], ['using', 'programming.'], ['programming.', 'so'], [
        'so', "don't"], ["don't", 'just'], ['just', 'learn'], ['learn', 'how'], ['how', 'to'], ['to', 'program'], ['program', 'build'], ['build', 'something'], ['something', 'that'], ['that', 'matters'], ['matters', 'build'], ['build', 'something'], ['something', 'that'], ['that', 'means'], ['means', 'something'], ['something', 'to'], ['to', 'you'], ['you', 'build'], ['build', 'something'], ['something', 'that'], ['that', 'solves'], ['solves', 'a'], ['a', 'problem'], ['problem', "you're"], ["you're", 'a'], ['a', 'problem'], ['problem', 'solver'], ['solver', 'a'], ['a', 'creator'], ['creator', 'an'], ['an', 'innovator'], ['innovator', "you're"], ["you're", 'not'], ['not', 'a'], ['a', 'programmer'], ['programmer', 'programming'], ['programming', 'is'], ['is', 'just'], ['just', 'a'], ['a', 'tool'], ['tool', 'in'], ['in', 'your'], ['your', 'arsenal'], ['arsenal', 'to'], ['to', 'build'], ['build', 'something'], ['something', 'amazing.'], ['amazing.', 'you'], ['you', 'can'], ['can', 'be'], ['be', 'analytical'], ['analytical', 'trade'], ['trade', 'of'], ['of', 'empathetic'], ['empathetic', 'and'], ['and', 'programming'], ['programming', 'allows'], ['allows', 'you'], ['you', 'to'], ['to', 'express'], ['express', 'those'], ['those', 'qualities'], ['qualities', 'of'], ['of', 'yourself'], ['yourself', 'but'], ['but', 'programming'], ['programming', 'in'], ['in', 'of'], ['of', 'itself'], ['itself', 'is'], ['is', 'nothing'], ['nothing', 'special'], ['special', "it's"], ["it's", 'like'], ['like', 'a'], ['a', 'pencil'], ['pencil', 'it'], ['it', 'can'], ['can', 'mark'], ['mark', 'stuff'], ['stuff', 'down'], ['down', 'if'], ['if', 'you'], ['you', 'press'], ['press', 'on'], ['on', 'the'], ['the', 'pencil'], ['pencil', 'hard'], ['hard', 'enough.'], ['enough.', "that's"], ["that's", 'it'], ['it', 'but'], ['but', 'with'], ['with', 'a'], ['a', 'pencil'], ['pencil', 'you'], ['you', 'can'], ['can', 'write'], ['write', 'novels'], ['novels', 'draw'], ['draw', 'beautiful'], ['beautiful', 'portraits'], ['portraits', 'build'], ['build', 'plans'], ['plans', 'for'], ['for', 'a'], ['a', 'skyscraper'], ['skyscraper', 'or'], ['or', 'anything'], ['anything', 'limited'], ['limited', 'only'], ['only', 'by'], ['by', 'your'], ['your', 'imagination.'], ['imagination.', 'you'], ['you', "don't"], ["don't", 'learn'], ['learn', 'how'], ['how', 'to'], ['to', 'program'], ['program', 'to'], ['to', 'get'], ['get', 'into'], ['into', 'google.'], ['google.', 'you'], ['you', 'learn'], ['learn', 'how'], ['how', 'to'], ['to', 'program'], ['program', 'to'], ['to', 'build'], ['build', 'something'], ['something', 'meaningful'], ['meaningful', 'something'], ['something', 'that'], ['that', 'helps'], ['helps', 'real'], ['real', 'people'], ['people', 'with'], ['with', 'real'], ['real', 'problems'], ['problems', "i'm"], ["i'm", 'partnered'], ['partnered', 'of'], ['of', 'ibm'], ['ibm', 'today'], ['today', 'to'], ['to', 'talk'], ['talk', 'about'], ['about', 'a'], ['a', 'global'], ['global', 'initiative'], ['initiative', 'called'], ['called', 'call'], ['call', 'for'], ['for', 'code'], ['code', 'which'], ['which', 'calls'], ['calls', 'for'], ['for', 'developers'], ['developers', 'to'], ['to', 'build'], ['build', 'something'], ['something', 'impactful'], ['impactful', 'and'], ['and', 'have'], ['have', 'a'], ['a', 'positive'], ['positive', 'change'], ['change', 'across'], ['across', 'the'], ['the', 'world'], ['world', 'through'], ['through', 'their'], ['their', 'code'], ['code', 'as'], ['as', 'you'], ['you', 'know'], ['know', 'there'], ['there', 'are'], ['are', 'hundreds'], ['hundreds', 'and'], ['and', 'hundreds'], ['hundreds', 'of'], ['of', 'natural'], ['natural', 'disasters'], ['disasters', 'every'], ['every', 'year'], ['year', 'like'], ['like', 'hurricanes'], ['hurricanes', 'and'], ['and', 'earthquakes'], ['earthquakes', 'floods'], ['floods', 'volcanoes'], ['volcanoes', 'and'], ['and', 'wildfires'], ['wildfires', 'those'], ['those', 'affect'], ['affect', 'numerous'], ['numerous', 'lies'], ['lies', 'and'], ['and', 'causes'], ['causes', 'tremendous'], ['tremendous', 'damage'], ['damage', 'to'], ['to', 'many'], ['many', 'families'], ['families', 'around'], ['around', 'the'], ['the', 'world'], ['world', 'if'], ['if', 'you'], ['you', 'strive'], ['strive', 'for'], ['for', 'real'], ['real', 'impact'], ['impact', 'we'], ['we', 'desperately'], ['desperately', 'need'], ['need', 'you'], ['you', 'this'], ['this', 'is'], ['is', 'why'], ['why', '2018'], ['2018', 'call'], ['call', 'for'], ['for', 'code'], ['code', 'global'], ['global', 'challenge'], ['challenge', 'is'], ['is', 'a'], ['a', 'competition'], ['competition', 'that'], ['that', 'ass'], ['ass', 'people'], ['people', 'like'], ['like', 'you'], ['you', 'to'], ['to', 'build'], ['build', 'solutions'], ['solutions', 'to'], ['to', 'improve'], ['improve', 'what'], ['what', 'we'], ['we', 'can'], ['can', 'do'], ['do', 'to'], ['to', 'reduce'], ['reduce', 'the'], ['the', 'destructive'], ['destructive', 'impact'], ['impact', 'of'], ['of', 'his'], ['his', 'natural'], ['natural', 'disasters'], ['disasters', 'we'], ['we', 'need'], ['need', 'all'], ['all', 'kinds'], ['kinds', 'of'], ['of', 'technologies.']]
    assert text.bigrams_extraction() == bigrams


def test_summarize(text):
    summary = """What kind of program should i write to build up my resume so why don't i like these questions because it feels like the people who are asking these questions they are learning how to program just for the sake of learning how to program. But i think it's a pay wasn't great i don't think a lot of you would spend hours and hours every night in front of the computer trying to learn how to code or to just debug your program sometimes we get caught up learning languages or doing coding challenges or fantasizing about you not working for these big tech companies that weforget why we wanted to learn programming in the first place. You don't learn how to program to get into google"""
    assert text.summarize(verbose=False) == summary

# def test_terms_frequency(text):
#     terms_sample = [{'today': 0.033145508783559825}, {'causes': 0.033145508783559825}, {'x': 0.033145508783559825}, {'innovator': 0.033145508783559825}, {'write': 0.09943652635067948}, {'get': 0.1325820351342393}, {'floods': 0.033145508783559825}, {'call': 0.06629101756711965}, {'qualities': 0.033145508783559825}, {'natural': 0.06629101756711965}, {'solver': 0.033145508783559825}, {'spend': 0.033145508783559825}, {'improve': 0.033145508783559825}, {'companies': 0.033145508783559825}, {'lies': 0.033145508783559825}, {'place.': 0.0}, {'novels': 0.033145508783559825}, {'money': 0.033145508783559825}, {'big': 0.033145508783559825}, {'reduce': 0.033145508783559825}, {'trying': 0.033145508783559825}, {'anything': 0.033145508783559825}, {'competition': 0.033145508783559825}, {'meaningful': 0.033145508783559825}, {'volcanoes': 0.033145508783559825}, {'like.': 0.0}, {'code': 0.16572754391779915}, {'take.': 0.0}, {'right': 0.06629101756711965}, {'families': 0.033145508783559825}, {'questions': 0.16572754391779915}, {'portraits': 0.033145508783559825}, {'mark': 0.033145508783559825}, {'change': 0.033145508783559825}, {'the.': 0.0}, {'analytical': 0.033145508783559825}, {'build': 0.2983095790520385}, {'speak': 0.033145508783559825}, {'every': 0.06629101756711965}, {'enough.': 0.0}, {'tool': 0.06629101756711965}, {'computer': 0.033145508783559825}, {'need': 0.16572754391779915}, {'open': 0.033145508783559825}, {'special': 0.033145508783559825}, {'would': 0.033145508783559825}, {'hard': 0.033145508783559825}, {'programming': 0.2651640702684786}, {'tool.': 0.0}, {'imagination.': 0.0}, {'first': 0.033145508783559825}, {'front': 0.033145508783559825}, {'damage': 0.033145508783559825}, {'saw': 0.033145508783559825}, {'people': 0.09943652635067948}, {'turn': 0.033145508783559825}, {'machine': 0.033145508783559825}, {'impactful': 0.033145508783559825}, {'means': 0.06629101756711965}, {'series': 0.033145508783559825}, {'across': 0.033145508783559825}, {'google.': 0.0}, {'empathetic': 0.033145508783559825}, {'feels': 0.033145508783559825}, {'technologies.': 0.0}, {'press': 0.033145508783559825}, {'year': 0.033145508783559825}, {'got': 0.033145508783559825}, {'nothing': 0.033145508783559825}, {'disasters': 0.06629101756711965}, {'sake': 0.033145508783559825}, {'think': 0.1325820351342393}, {'limited': 0.033145508783559825}, {'calls': 0.033145508783559825}, {'make': 0.033145508783559825}, {'refrigerator': 0.033145508783559825}, {'exact': 0.033145508783559825}, {'affect': 0.033145508783559825}, {'skyscraper': 0.033145508783559825}, {'tech': 0.033145508783559825}, {'something': 0.2651640702684786}, {'things.': 0.0}, {'solutions': 0.033145508783559825}, {'ibm': 0.033145508783559825}, {'pay': 0.033145508783559825}, {'example': 0.033145508783559825}, {"i'm": 0.0}, {'world': 0.06629101756711965}, {'resume': 0.033145508783559825}, {'hours': 0.06629101756711965}, {'kind': 0.06629101756711965}, {
#         'arsenal': 0.033145508783559825}, {'working': 0.033145508783559825}, {'pencil': 0.09943652635067948}, {'asking': 0.033145508783559825}, {'kinds': 0.033145508783559825}, {'ass': 0.033145508783559825}, {'lot': 0.06629101756711965}, {'positive': 0.033145508783559825}, {'challenge': 0.033145508783559825}, {'global': 0.06629101756711965}, {'steps': 0.033145508783559825}, {'problems': 0.033145508783559825}, {'amazing.': 0.0}, {'trade': 0.033145508783559825}, {'impact': 0.06629101756711965}, {'great': 0.033145508783559825}, {'built': 0.033145508783559825}, {'allows': 0.033145508783559825}, {'language': 0.033145508783559825}, {'tremendous': 0.033145508783559825}, {'actually': 0.033145508783559825}, {'things': 0.09943652635067948}, {'giving': 0.033145508783559825}, {'helps': 0.033145508783559825}, {'challenges': 0.033145508783559825}, {'initiative': 0.033145508783559825}, {'program.': 0.0}, {'certain': 0.033145508783559825}, {'become': 0.033145508783559825}, {'perform': 0.033145508783559825}, {'wildfires': 0.033145508783559825}, {'destructive': 0.033145508783559825}, {'actions': 0.033145508783559825}, {'express': 0.033145508783559825}, {'languages': 0.06629101756711965}, {'desperately': 0.033145508783559825}, {'programmer': 0.06629101756711965}, {'instructions': 0.033145508783559825}, {'matters': 0.033145508783559825}, {'problem': 0.06629101756711965}, {'talk': 0.033145508783559825}, {'amazing': 0.06629101756711965}, {'strive': 0.033145508783559825}, {'weforget': 0.033145508783559825}, {'automate': 0.033145508783559825}, {'want': 0.033145508783559825}, {'plans': 0.033145508783559825}, {'called': 0.033145508783559825}, {'door.': 0.0}, {'learning': 0.09943652635067948}, {'difference': 0.033145508783559825}, {'2018': 0.033145508783559825}, {"they're": 0.0}, {'using': 0.033145508783559825}, {'program': 0.2983095790520385}, {'light': 0.033145508783559825}, {'programming.': 0.0}, {'boring.': 0.0}, {'fantasizing': 0.033145508783559825}, {'real': 0.09943652635067948}, {'partnered': 0.033145508783559825}, {'company': 0.033145508783559825}, {'hundreds': 0.06629101756711965}, {'ask': 0.033145508783559825}, {'solves': 0.033145508783559825}, {'earthquakes': 0.033145508783559825}, {'around': 0.033145508783559825}, {"what's": 0.0}, {'beautiful': 0.033145508783559825}, {'many': 0.06629101756711965}, {'stuff': 0.033145508783559825}, {'you.': 0.0}, {'hurricanes': 0.033145508783559825}, {'developers': 0.033145508783559825}, {'like': 0.19887305270135897}, {'draw': 0.033145508783559825}, {'creator': 0.033145508783559825}, {'numerous': 0.033145508783559825}, {"that's": 0.0}, {'makes': 0.033145508783559825}, {'know': 0.033145508783559825}, {'debug': 0.033145508783559825}, {'sometimes': 0.033145508783559825}, {'night': 0.033145508783559825}, {'caught': 0.033145508783559825}, {'wanted': 0.06629101756711965}, {'coding': 0.033145508783559825}, {'learn': 0.23201856148491878}, {'b': 0.033145508783559825}, {'it.': 0.0}]
#     assert text.term_frequency() == terms_sample


def test_remove_punctuation():
    text_w_puncts = "a.i,o;u'e"
    assert len(Text.remove_punctuation(text=text_w_puncts)) == 5


def test_remove_numbers():
    text_w_numbers = "a2i3o4u5e"
    assert len(Text.remove_numbers(text=text_w_numbers)) == 5


def test_remove_non_unicode():
    text_w_non_unicoode = "Ceñía"
    assert Text.noise_removal(text=text_w_non_unicoode) == 'Cea'


def test_sentiment_vader_test(text):
    assert text.sentiment_analysis(method='vader') == {'neg': 0.099, 'neu': 0.776, 'pos': 0.125, 'compound': 0.9217}


def test_remove_non_unicode():
    sample = Text.remove_non_unicode('Ceñía')
    assert sample == 'Cea'


def test_noise_removal():
    sample = Text.noise_removal("Hi1 I'm    Victor Ceñía")
    assert sample == 'hi im victor cea'


def test_normalization():
    assert Text("I'm coding it to be the best application.").normalization(
    ) == "im code best applic"


def test_text_stemming():
    sample = Text("I'm coding it to be the best application.")
    assert "i'm code it to be the best application." == sample.stemming()


def test_sentence_similarity():
    sample = Text.sentence_similarity('hello beautiful world', 'hello world')
    assert sample == 0.8598892366800223
