# -*- coding: utf-8 -*-
"""Brown Corpus raw, words, sents, categories.

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/17dvmih9_LQsidb3e9Uga9W1nWa2-m9zA
"""

import nltk
nltk.download('brown')
from nltk.corpus import brown
print ('File ids of brown corpus\n',brown.fileids())

'''NLTK includes a small selection of texts from the Project brown electronic text
archive, which contains some 25,000 free electronic books, hosted at
http://www.brown.org/. We begin by getting the Python interpreter to load the NLTK
package, then ask to see nltk.corpus.brown.fileids(), the file identifiers in this corpus:'''

ca01 = brown.words('ca01')
print('\nca01 has following words:\n',ca01)

print('\nca01 has',len(ca01),'words')

print ('\n\nCategories or file in brown corpus:\n')
print (brown.categories())

print ('\n\nStatistics for each text:\n')
print('AvgWordLen\tAvgSentenceLen\tno.ofTimesEachWordAppearsOnAvg\t\tFileName')
for fileid in brown.fileids():
    num_chars = len(brown.raw(fileid))
    num_words = len(brown.words(fileid))
    num_sents = len(brown.sents(fileid))
    num_vocab = len(set([w.lower() for w in brown.words(fileid)]))
    print (int(num_chars/num_words),'\t\t\t',
    int(num_words/num_sents),'\t\t\t',int(num_words/num_vocab),'\t\t\t', fileid)

