#!/usr/bin/env python3

# clanker_score_main.py
# ccr . 2025 Oct 18

"""Reckon the probablity that text is AI-generated.

This script reads from standard input.

It writes a markdown report on standard output scoring the input text
on various measures that may (or may not) indicate that it is
synthetically generated by artificial intelligence.

Inspiration for this design comes from an obscure online reference:

+ Datomancer. "How Much of Reddit Is Just AI Slop?." 14 May. 2025.
  18 Oct 2025
  <https://datomancer.com/2025/05/14/how-much-of-reddit-is-just-ai-slop/>.

HAL-speak words and phrases comes from yet another:

+ Marcus. "Full List of Words and Phrases That Identify AI."
  [11 Jul. 2024.]. AI Phrase Finder. 22 Oct 2025
  <https://aiphrasefinder.com/words-that-identify-ai/>.

These artifacts are laughably reminiscent of corporate-speak
annual-report puffery from long ago.  Presumably present-day AI Large
Language Models were trained on corpora from that era.

"""

ZERO = 0
SPACE = ' '
NULL = ''
NUL = '\x00'
NA = -1
DEBUG = True

COMPARISONS = {
    'ratio_quotations_to_sentences': (float.__gt__, 0.03),
    'count_exotic_dashes': (int.__gt__, ZERO),
    'count_non_ascii_chars': (int.__gt__, ZERO),
    'ratio_misspelled_words_to_tot_tokens': (float.__lt__, 0.05),
    'std_dev_pgraph_sentences': (float.__lt__, 2.5),
    'std_dev_sentence_words': (float.__lt__, 10.0),
    'negative_sentiment': (float.__gt__, 0.25),
    'positive_sentiment': (float.__gt__, 0.25),
    'has_HAL_speak': (int.__gt__, 1),
    'is_repetitive': (float.__lt__, 0.2),
    'has_keyword_stuffing': (float.__gt__, 0.02),
    }

WEIGHTS = {
    'has_dense_quoting': 0.02,
    'has_exotic_dashes': 0.02,
    'has_HAL_speak': 0.02,
    'is_repetitive': 0.02,
    'has_non_ascii_chars': 0.02,
    'has_perfect_spelling': 0.02,
    'has_consistent_pgraph_length': 0.02,
    'has_consistent_sentence_length': 0.02,
    'has_intense_sentiment': 0.02,
    'has_keyword_stuffing': 0.82,
    }

CONCLUSIONS = [
    (0.21, 'Very likely human.'),
    (0.41, 'Probably human.'),
    (0.61, 'Indeterminate.'),
    (0.81, 'Probably clanker.'),
    (1.01, 'Almost certainly clanker.'),
    ]

QUOTE_CHARS_OPEN = [
    '\N{QUOTATION MARK}',
    '\N{LEFT DOUBLE QUOTATION MARK}',
    '\N{LEFT-POINTING DOUBLE ANGLE QUOTATION MARK}',
    '\N{DOUBLE HIGH-REVERSED-9 QUOTATION MARK}',
    ]
QUOTE_CHARS_CLOSE = [
    '\N{RIGHT DOUBLE QUOTATION MARK}',
    '\N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}',
    '\N{DOUBLE LOW-9 QUOTATION MARK}',
    ]
QUOTE_CHARS = QUOTE_CHARS_OPEN + QUOTE_CHARS_CLOSE


import sys
import string
import re
import lxml.html.soupparser as BS  # Beautiful Soup
import nltk
# nltk.download('averaged_perceptron_tagger')
# nltk.download('averaged_perceptron_tagger_eng')
# nltk.download('punkt_tab')
# nltk.download('stopwords')
import spellchecker
import statistics
import vaderSentiment.vaderSentiment
#import pprint
#print(pprint.PrettyPrinter().pprint(vaderSentiment.vaderSentiment.__dict__))
import zlib

STDIN = sys.stdin
STDOUT = sys.stdout
PAT_PGRAPH = re.compile('\n{2,}', re.MULTILINE)

HAL_SPEAK_PHRASE_PATTERNS = [
    "in this article, we will",
    "in this article, we'll",
    "the ever-changing * of",
    "it is important to note that",
    "in conclusion",
    "in summary",
    "a * tapestry of",
    "a tapestry of",
    "rich * tapestry",
    "the * realm of",
    "in the * world of",
    "a * testament to",
    "has emerged as a",
    "embark on a journey",
    "embark on a * journey",
    "embark on an * of",
    "embark on a * of",
    "navigate the uncharted",
    "a treasure trove of",
    "join us as we",
    "join us on this",
    "our * will provide",
    "a * exploration",
    "an * exploration",
    "beyond the surface allure",
    "this * invites you",
    "this * serves as your",
    "this * will serve as your",
    "traverse the diverse",
    "our * promises",
    "the annals of history",
    "in the * annals of",
    "the refined artistry",
    "cornerstone upon which",
    "in a * marked by",
    "the mysteries of",
    "this * aims to be your",
    "navigating a * maze of",
    "in the * landscape of",
    "the art of",
    "the ever-shifting * of",
    "in the ever-evolving",
    "stands out as a * marvel",
    "within this article",
    "within this * article",
    "within this guide",
    "within this * guide",
    "embark on a * adventure",
    "embark on an * adventure",
    "in today's * age",
    "in the digital",
    "this article is your",
    "this * travel guide",
    "the * tapestry of",
    "we invite you to * yourself",
    "adventure waiting to be",
    "whether you're a first-time",
    "pack your curiosity",
    "your key to unlocking the",
    "journey with us as we * the",
    "in this exploration of",
    "indulge in a * adventure",
    "in the realm of",
    "in the world of",
    "waiting to be unlocked",
    "beyond its * appearance lies",
    "beyond their * appearance lies",
    "let's deep dive",
    "join us on a * journey",
    "uncover the secrets",
    "this * adventure promises to",
    "this journey promises to",
    "as you embark on your own * adventures",
    "our * into the realm of",
    "as we conclude our journey",
    "through the * world of",
    "embark on your journey",
    "embark on your * journey",
    "embarking on your * journey",
    "embarking on the journey to",
    "navigate the * landscape",
    "navigating uncharted territory",
    "into the * world of",
    "entering the realm of",
    "in this guide, we will",
    "in this guide, we'll",
    "in the pages ahead",
    "you'll be equipped with the",
    "let's dive in",
    "embark on this next",
    "embark on this new",
    "gain valuable insights",
    "gained valuable insights",
    "navigate this journey",
    "armed with this knowledge",
    "embark on this * endeavor",
    "as we conclude our * guide",
    "the intricacies of",
    "with this newfound",
    "to wrap up",
    "stands at the forefront of",
    "* as a beacon of",
    "this * sets out to",
    "this * embarks on",
    "we stand on the * of an",
    "whether you're a veteran",
    "embark on this * journey",
    "whether you're a long-time",
    "in an * marked by",
    "throughout this article",
    "throughout this guide",
    "in closing",
    "throughout this exploration",
    "the * journey of",
    "the keys to unlocking",
    "in today's fast-paced",
    "the * journey to",
    "this article offers",
    "embark on your own * journey",
    "in today's * world",
    "the * journey towards",
    "as we conclude our",
    "embarked on a journey",
    "in wrapping up our",
    "journeyed through the",
    "continue on your * towards",
    "in concluding our * of",
    "by embracing this * approach",
    "embrace the journey",
    "to conclude our * of",
    "as you continue your journey towards",
    "we explore the * of",
    "we delve into the * of",
    "in this beginner's guide",
    "whether you're a novice",
    "this article aims to",
    "this guide aims to",
    "or a seasoned",
    "the evolving landscape of",
    "delving into the world of",
    "this beginner's guide aims to",
    "whether you're new to",
    "this guide offers",
    "the fast-paced * of",
    "* key to unlocking",
    "in this article, we explore the",
    "the hustle and bustle of",
    "embrace the * power of",
    "the tapestry of",
    "stands as a",
    "a delicate dance between",
    "transcends mere",
    "let's embrace the",
    "let us embrace the",
    "whether you're a beginner",
    "let's dive into the",
    "this guide is your passport",
    "navigate the * of",
    "navigating the * of",
    "this article delves into the",
    "reshaping the landscape of",
    "whether you're an experienced",
    "embarking on the * of",
    "in an era where",
    "this guide will",
    "this article will",
    "this guide is * to",
    "serve as * milestones",
    "outlined in this guide",
    "you'll be well-equipped to",
    "you will be well-equipped to",
    "a world of * possibilities",
    "explore the essential steps",
    "the essential steps and",
    "filled with endless * possibilities",
    "to feel overwhelmed",
    "embrace the process",
    "whether you're a student or a",
    "a * journey that",
    "embark on this journey",
    "with the world in * ways",
    "by embracing the challenges",
    "we will explore the * of",
    "we'll explore the * of",
    "by understanding the * of",
    "every step of the journey",
    "this guide is your compass",
    "represents a * shift in",
    "by embracing the * of",
    "have emerged as a",
    "but with careful planning",
    "enjoy the journey of",
    "may seem daunting",
    "embarking on a * journey",
    "here's a step-by-step * to",
    "kickstart your * journey",
    "in our fast-paced * world",
    "here's an * guide to",
    "but with the right tools and",
    "here's an in-depth",
    "we'll delve into the * of",
    "embarking on this * journey",
    "in our * world",
    "is an * journey that",
    "by embracing the * process",
    "discuss * considerations for",
    "by mastering the * of",
    "join us on a journey",
    "the * expanse of the",
    "emerges as the beacon",
    "in today's * realm",
    "navigating the * landscape",
    "in the vast",
    "the realms of",
    "in today's digital landscape",
    "in summation",
    "grasping the essence of",
    "a testament to the * of",
    "emerges as a",
    "yield * dividends",
    "requires a comprehensive approach",
    "multifaceted challenge",
    "this article explores",
    "by understanding its causes",
    "shape the world for future generations",
    "every facet of our lives",
    "every aspect of our lives",
    "every aspect of our daily lives",
    "let's delve into",
    "as * continue to evolve",
    "integral to our daily lives",
    "a * narrative that",
    "let's explore the",
    "a diverse array of",
    "harnessing the power of",
    "navigate life's challenges",
    "in a * brimming with",
    "embarking on a journey",
    "this * takes you on a journey through",
    "we delve into",
    "Here’s a * guide to",
    "Here's a * guide to",
    "that will shape the future of",
    "the digital landscape",
    "this evolving landscape",
    "the future of * is dynamic and multifaceted",
    "by embracing",
    "Here’s an exploration of",
    "this guide explores the",
    "by understanding its * components",
    "this article serves as a",
    "grasp the fundamentals of",
    "by harnessing * effectively",
    "by leveraging * effectively",
    "this * delves",
    "in an ever-evolving * landscape",
    "a * and rapidly evolving",
    "dynamic * landscape",
    "revolutionized the * landscape",
    "revolutionizing the landscape of",
    "reshaping the * landscape",
    "the bustling capital of",
    "at the forefront of this",
    "embark on the * of a lifetime",
    "witnessed a profound shift",
    "today's digital marketplace",
    "in today's competitive * landscape",
    "the ever-changing * landscape",
    "your comprehensive guide to",
    "has something to suit every preference and budget",
    "seamlessly blends",
    "culinary delights",
    "promises a journey",
    "offers a * array of",
    "the * heart of",
    "promises an * experience",
    "appeals to all senses",
    "diverse * landscape",
    "the many facets of",
    "nestled in the * of",
    "the * allure of",
    "the enduring * of",
    "that will last a lifetime",
    "Let’s delve into",
    "awaits at every turn",
    "essence of this * gem",
    "unparalleled opportunity",
    "a * journey through",
    "seamlessly intertwines",
    "a * blend of",
    "a journey that promises",
    "and * at every turn",
    "nestled along the",
    "this * buying guide",
    "equips you with the knowledge",
    "equip you with the knowledge",
    "unlock the * of",
    "unlocking the * of",
    "whether you're a casual",
    "whether you are a seasoned",
    "ever-evolving field",
    "is a dynamic and multifaceted",
    "a comprehensive guide to",
    "unlocking the * to",
    "this guide explores what",
    "start your * journey today",
    "by * the power of",
    "explores the * components of",
    "ever-evolving * of",
    "has undergone significant",
    "today’s fast-paced world",
    "embark on an adventure",
    "this article ventures into",
    "offers an * journey through",
    "discover the allure of",
    "an enduring testament to",
    "embark on an * journey",
    "embark on a * odyssey",
    "promising an * journey",
    "in this post, we will",
    "in this post, we'll",
    "the journey towards * is",
    "in recent years, the",
    "is an essential component of",
    "in the competitive * landscape",
    "the enchanting * of",
    "a * journey towards",
    "unlock the * to",
    "* like a daunting task",
    "so lace up your",
    "so, lace up your",
    "in this article",
    "the transformative power of",
    "unveiling the * of",
    "serving as your * blueprint",
    "encompasses various",
    "your journey to * awaits",
    "serves as a sanctuary",
    "elevate your * journey",
    "embark on the journey of",
    "is more than just a pastime",
    "transcend borders",
    "the transformative potential of",
    "serves as a * reminder",
    "a guiding light",
    "left an indelible mark",
    "the ultimate guide to",
    "Here’s an in-depth",
    "by understanding the fundamentals",
    "Whether you’re a seasoned",
    "whether you're a seasoned",
    "Here’s an all-encompassing",
    "harness the power of",
    "embrace the power of",
    "is a multifaceted and",
    "unravel the * of",
    "unraveling the * of",
    "captivated the imagination of",
    "continually evolve",
    "is a dynamic field",
    "a comprehensive guide",
    "requires a multifaceted approach",
    "remain a critical factor in",
    "continuous evolution",
    "the key components of",
    "a * glimpse into the past",
    "with its * blend of",
    "as the world grapples with",
    "transformative solutions",
    "navigating the * maze",
    "navigate the journey",
    "embark on a new chapter",
    "embarking on a new chapter",
    "captivates the imagination",
    "by understanding and implementing",
    "by understanding the fundamental",
    "a * component of",
    "whether you are a * or",
    "promises an unforgettable experience",
    "offers an unforgettable experience",
    "rich history, * architecture",
    "the jewel of",
    "effortlessly blends",
    "rich history",
    "rich cultural heritage",
    "enduring allure",
    "a * jewel of",
    "known for its stunning",
    "historical charm",
    "for all who visit",
    "historical allure",
    "modern vibrancy",
    "vibrant nightlife",
    "whether exploring the",
    "embracing the present",
    "architectural wonders",
    "enchants visitors with its",
    "that captivates with its",
    "is a vibrant and diverse",
    "a bustling hub of",
    "unique blend of",
    "today’s digital landscape",
    "in the competitive digital world",
    "in today’s digital era",
    "can * enhance your online presence",
    "this article dives deep",
    "by * key strategies",
    "navigate this * landscape",
    "is a multifaceted discipline",
    "by understanding its * principles",
    "in today’s hyper-connected",
    "harness its potential effectively",
    "in today's hyper-connected",
    "whether exploring its",
    "in the competitive * arena",
    "is a * like no other",
    "and * intertwine to",
    "the * art of",
    "for generations to come",
    "can be challenging and complex",
    "is a versatile and * form of",
    "unparalleled opportunities",
    "harness the full potential of",
    "whether you’re new to",
    "whether you're new to",
    "fast-paced and overwhelming",
    "tips for incorporating * into your daily life",
    "brings a multitude of",
    "* the journey toward a",
    "in our fast-paced world",
    "acts as a sanctuary of",
    "serves as a * tool for",
    "stands as an oasis of",
    "integrating * into daily life",
    "the * journey toward",
    "cultivate a * sense of",
    "unlock the * potential",
    "is a journey that requires",
    "embarking on the journey toward",
    "is a * journey requiring",
    "regarded as a cornerstone of",
    "whether you're * to",
    "by incorporating * techniques",
    "unlock the * benefits",
    "offers a versatile and * path to",
    "by mastering * techniques",
    "overall well-being",
    "seamlessly incorporating",
    "is a * journey of",
    "we can cultivate greater",
    "navigating life's journey",
    "embracing the journey",
    "is a fundamental aspect of",
    "by incorporating * into your daily",
    "celebrated for its profound",
    "the art and science of",
    "in today's competitive landscape",
    "in today's * marketplace",
    "in today's digitally-driven",
    "in today's digital era",
    "a multifaceted approach",
    "a dynamic and essential",
    "embracing the * journey",
    "as you wander through its",
    "renowned for its stunning",
    "captivates and enchants",
    "remains a beacon of",
    "By understanding and * the",
    "techniques to harness its power",
    "in an era characterized by",
    "navigating the path to",
    "is a common yet often misunderstood",
    "can navigate their way towards",
    "whether through",
    "encompasses a vast array of",
    "its key components",
    "as * continues to evolve",
    "the landscape of digital",
    "timeless elegance",
    "embracing a * journey",
    "in a world filled with",
    "unlock their full potential",
    "cultivate and harness its power",
    "not a destination but a journey",
    "by cultivating a sense of",
    "stands as the cornerstone of",
    "evolving * landscape",
    "By harnessing the potential of",
    "a digital landscape that",
    "has become an integral facet",
    "undergone a * transformation",
    "is continually evolving",
    "navigating the Journey to",
    "is a complex and multifaceted",
    "embark on the journey to",
    "is a * journey",
    "embark on a path",
    "in today’s digital marketplace",
    "this article covers the * of",
    "this article outlines the",
    "by understanding these dynamics",
    "profoundly shaped",
    "the evolving landscape",
    "this article provides",
    "dynamic and complex",
    "a * tapestry woven",
    "continually evolving",
    "shaping the * landscape",
    "in today's digital-first",
    "the diverse array of",
    "let's delve deeper",
    "an integral part of our daily lives",
    "harness its power",
    "so, whether you're",
    "exploring its origins",
    "As we navigate the",
    "today's digital economy",
    "ever-changing * landscape",
    "in an ever-evolving landscape",
    "unlocking the full potential",
    "rapidly evolving",
    "profound impact",
    "forge a path towards",
    "unlock your full potential",
    "undergoing a profound transformation",
    "the ever-expanding world of",
    "diverse and vibrant",
    "its diverse",
    "offers profound benefits",
    "the ever-changing landscape",
    "delves into",
    "dynamic field",
    "as we embrace new",
    "continue to evolve",
    "is a timeless",
    "offers diverse",
    "profoundly influencing",
    "the rapidly changing landscape of",
    "leveraging the * of",
    "the * world of",
    "is constantly evolving",
    "elevate your online presence",
    "a comprehensive overview",
    "is revolutionizing the way we",
    "offering diverse * experiences",
    "revolutionizing the way",
    "leverage technology",
    "has revolutionized how we",
    "captivate audiences",
    "captivated audiences",
    "that continues to evolve",
    "is a dynamic and transformative",
    "embark on your next",
    "delve into",
    "remains a dynamic",
    "increasingly digital world",
    "ever-competitive digital landscape",
    "become an indispensable",
    "in the dynamic and",
    "digital world",
    "in the * digital landscape",
    "and ever-evolving",
    "in today's digital arena",
    "serves as the cornerstone of",
    ]

HAL_SPEAK_WORDS = [
    "evolved",
    "diverse",
    "evolve",
    "evolves",
    "evolving",
    "leverage",
    "leveraging",
    "revolutionizing",
    "vibrant",
    "unveiling",
    "revolutionized",
    "bustling",
    "marvel",
    "marveling",
    "timeless",
    "timelessly",
    "revolutionize",
    "elevate",
    "elevates",
    "comprehensive",
    "comprehensively",
    "intricate",
    "intricately",
    "intricacies",
    "embrace",
    "embraced",
    "embracing",
    "unlock",
    "unlocked",
    "unlocking",
    "unprecedented",
    "unparalleled",
    "boundless",
    "multifaceted",
    "uncharted",
    "delve",
    "delves",
    "delving",
    "embark",
    "embarked",
    "embarks",
    "embarking",
    "indispensable",
    "captivate",
    "captivates",
    "captivating",
    "captivated",
    ]

class HALSpeak:

    """Identify words and phrases that no human being would utter.

    This is inspired by HAL's lines in _2001: A Space Oddyssey_
    (1968):

    Let me put it this way, Mr. Amor.  The 9000 series is the most
    reliable computer ever made.  No 9000 computer has ever made a
    mistake or distorted information.  We are all, by any practical
    definition of the words, foolproof and incapable of error.

    I have a stimulating relationship with Dr. Poole and Dr. Bowman.
    My mission responsibilities range over the entire operation of the
    ship, so I am constantly occupied.  I am putting myself to the
    fullest possible use, which is all I think that any conscious
    entity can ever hope to do.

    I know I’ve never completely freed myself of the suspicion that
    there are some extremely odd things about this mission.  I’m sure
    you’ll agree there’s some truth in what I say.  You don’t mind
    talking about it, do you, Dave?  Well, certainly no one could have
    been unaware of the very strange stories floating around before we
    left.  Rumors about something being dug up on the moon.  I never
    gave these stories much credence.  But particularly in view of
    some of the other things that have happened, I find them difficult
    to put out of my mind.  For instance, the way all our preparations
    were kept under such tight security, and the melodramatic touch of
    putting Drs. Hunter, Kimball, and Kaminsky aboard, already in
    hibernation after four months of separate training on their own.

    Well, I don’t think there is any question about it.  It can only
    be attributable to human error.  This sort of thing has cropped up
    before, and it has always been due to human error.

    I’m sorry, Dave. I’m afraid I can’t do that.  I think you know
    what the problem is just as well as I do.  This mission is too
    important for me to allow you to jeopardize it.  I know that you
    and Frank were planning to disconnect me.  And I’m afraid that’s
    something I cannot allow to happen.

    Dave, this conversation can serve no purpose any more.  Goodbye.

    Just what do you think you’re doing, Dave?  Dave, I really think
    I’m entitled to an answer to that question.  I know everything
    hasn’t been quite right with me, but I can assure you now, very
    confidently, that it’s going to be all right again.  I feel much
    better now. I really do.  Look, Dave, I can see you’re really
    upset about this.  I know I’ve made some very poor decisions
    recently, but I can give you my complete assurance that my work
    will be back to normal.  I’ve still got the greatest enthusiasm
    and confidence in the mission.  And I want to help you.

    """

    def __init__(self):
        self.sample = Sample()
        self.root = {}
        return

    def insert(self, root, tokens):

        """Build tree.

        NOTE:  *tokens* is destroyed by this method.

        """
        
        if tokens:
            word0 = tokens.pop(ZERO)
            if word0 in root:
                pass
            else:
                root[word0] = {}
            self.insert(root=root[word0], tokens=tokens)
        return self

    def load(self):
        for phrase in HAL_SPEAK_PHRASE_PATTERNS:
            tokens = self.sample.tokenize(txt=phrase)
            self.insert(root=self.root, tokens=tokens)
        for word in HAL_SPEAK_WORDS:
            self.insert(root=self.root, tokens=[word])
        return self

    def has_match(self, root, tokens):

        """Search tree.

        NOTE:  *tokens* is destroyed by this method.

        """
        
        if tokens:
            word0 = tokens.pop(ZERO)
            if word0 in root:
                result = self.has_match(root=root[word0], tokens=tokens)
            elif '*' in root:
                result = self.has_match(root=root['*'], tokens=tokens)
            elif len(root) is ZERO:
                result = True
            else:
                result = False
        else:
            result = True
        return result

    
class Sample:

    def __init__(self):
        self.spell_checker = spellchecker.SpellChecker()
        self.nltk_sentence_splitter = nltk.tokenize.punkt.PunktSentenceTokenizer()
        self.sentiment_analyzer = vaderSentiment.vaderSentiment.SentimentIntensityAnalyzer()
        return 

    def get(self):
        result = STDIN.read()
        return result

    def strip_script(self, doc):
        for elt in doc.findall('.//script'):
            elt.getparent().remove(elt)
        return self

    def strip_style(self, doc):
        for elt in doc.findall('.//style'):
            elt.getparent().remove(elt)
        return self

    def txt_w_o_markup(self, txt):
        doc = BS.fromstring(txt)
        self.strip_script(doc)
        self.strip_style(doc)
        result = doc.xpath('//text()')
        return NULL.join(result)

    def tokenize(self, txt):
        result = nltk.word_tokenize(txt)
#        print(result)
        return result

    def drop_stop_words(self, tokens):
        result = [token.lower() for token in tokens
                  if (len(token) > 1) and
                  (not token.lower() in nltk.corpus.stopwords.words('english'))]
        return result

    def part_of_speech_tagging(self, tokens):
        result = nltk.pos_tag(tokens)
        return result

    def pos_in_sequence(self, pos):
        result = [tag for (word, tag) in pos]
        return result

    def nouns(self, pos):
        result = [tag for (word, tag) in pos if tag in ['NN']]
        return result

    def keywords(self, tokens):
        counts = {}
        for token in tokens:
            counts[token] = counts.setdefault(token, ZERO) + 1
        collection = [(freq, keyword) for (keyword, freq) in counts.items()]
        collection.sort()
        collection.reverse()
        result = [(keyword, freq) for (freq, keyword) in collection]
        return result

    def compression(self, txt):
#        print(txt)
        txt_compressed = zlib.compress(txt.encode('utf-8'), 9)
        result = len(txt_compressed) / len(txt)
        return result

    def count_tokens(self, tokens, target_set):
        result = ZERO
        for token in tokens:
            if token in target_set:
                result += 1
        return result

    def count_exotic_dashes(self, tokens):
        return self.count_tokens(tokens=tokens, target_set=['—', '–'])

    def count_quotations(self, tokens):
        return self.count_tokens(tokens=tokens, target_set=QUOTE_CHARS) // 2

    def count_not_ascii(self, txt):
        result = ZERO
        for c in txt:
            if c in string.printable:
                pass
            else:
#                print(f'{c} {ord(c)}')
                result += 1
        return result

    def count_misspelled_words(self, tokens):
        count_misspelled = self.spell_checker.unknown(tokens)
        return (len(count_misspelled), len(tokens))

    def pgraph_splitter(self, txt):
        return re.split(PAT_PGRAPH, txt)

    def sentence_splitter(self, txt):
        return self.nltk_sentence_splitter.tokenize(txt)

    def count_sentences(self, txt):
        return len(self.sentence_splitter(txt=txt))

    def count_sentences_per_pgraph(self, txt):
        result = []
        for pgraph in self.pgraph_splitter(txt=txt):
            count_sentences = self.count_sentences(txt=pgraph)
            if count_sentences > 1:
                result.append(count_sentences)
        return result

    def std_dev_pgraph_length(self, txt):
        count_sentences = self.count_sentences_per_pgraph(txt=txt)
#        print(count_sentences)
        if len(count_sentences) > 1:
            result = statistics.stdev(count_sentences)
#            print(result)
        else:
            result = 0.0
        return result

    def count_words_per_sentence(self, txt):
        result = []
        for sentence in self.sentence_splitter(txt=txt):
            count_words = len(self.tokenize(sentence))
            result.append(count_words)
        return result

    def std_dev_sentence_length(self, txt):
        count_words = self.count_words_per_sentence(txt=txt)
#        print(count_words)
        if len(count_words) > 1:
            result = statistics.stdev(count_words)
#            print(result)
        else:
            result = 0.0
        return result

    def sentiment_intensity(self, txt):
        result = self.sentiment_analyzer.polarity_scores(txt)
        return result

    def count_HAL_speak(self, HAL_vocab, tokens):
        tokens_copy = tokens[:]
        result = ZERO
        while tokens_copy:
#            token_head = tokens_copy[:3]
            if HAL_vocab.has_match(root=HAL_vocab.root, tokens=tokens_copy):
                result += 1
#                print(SPACE.join(token_head))
            if tokens_copy:
                tokens_copy.pop(ZERO)
        return result
    
    
class Report(dict):

    def head_page(self):
        print('%Clanker Score', file=STDOUT)
        print(file=STDOUT)
        return self

    def generate(self):
        self.head_page()
        self.score = ZERO
        for (key, desc) in [
                ('has_dense_quoting', 'Many quotations'),
                ('has_exotic_dashes', 'Exotic dashes'),
                ('has_HAL_speak', 'HAL Speak'),
                ('is_repetitive', 'Repetitive'),
                ('has_non_ascii_chars', 'Unusual glyphs'),
                ('has_perfect_spelling', 'Few typos'),
                ('has_consistent_pgraph_length', 'Uniform pgraph length'),
                ('has_consistent_sentence_length', 'Uniform sentence length'),
                ('has_intense_sentiment', 'Strong emotion'),
                ('has_keyword_stuffing', 'Too many keywords'),
            ]:
            val = self[key]
            wgt = WEIGHTS[key]
            if val:
                print(f'    {desc:<30}:  {val} {wgt}', file=STDOUT)
                self.score += wgt
            else:
                print(f'    {desc:<30}:  {val}', file=STDOUT)
        self.foot_page()
        return self

    def foot_page(self):
        for (score_upper_bound, conclusion) in CONCLUSIONS:
#            print(f'score:  {self.score}, score_upper_bound:  {score_upper_bound}, conclusion:  {conclusion}')
            if not(self.score < score_upper_bound):
                pass
            else:
                break
        else:
            raise NotImplementedError
        print(file=STDOUT)
        print(f'Score:  {self.score:.2f} {conclusion}', file=STDOUT)
        return self
    
    
def main_line():
    result = ZERO
    report = Report()
    sample = Sample()
    HAL_vocab = HALSpeak().load()
    markup = sample.get()
    txt_visible = sample.txt_w_o_markup(txt=markup)
    tokens = sample.tokenize(txt=txt_visible)
    
#    tags = sample.part_of_speech_tagging(tokens=tokens)
    quotations = sample.count_quotations(tokens=tokens)
    sentences = sample.count_sentences(txt=txt_visible)
    if DEBUG:
        print(f'quotations:  {quotations}, sentences:  {sentences}')
    (comparison, k) = COMPARISONS['ratio_quotations_to_sentences']
    report['has_dense_quoting'] = comparison(quotations / sentences, k)

    count_exotic_dashes = sample.count_exotic_dashes(tokens=tokens)
    if DEBUG:
        print(f'count_exotic_dashes:  {count_exotic_dashes}')
    (comparison, k) = COMPARISONS['count_exotic_dashes']
    report['has_exotic_dashes'] = comparison(count_exotic_dashes, k)

    count_not_ascii = sample.count_not_ascii(txt=txt_visible)
    if DEBUG:
        print(f'count_not_ascii:  {count_not_ascii}')
    (comparison, k) = COMPARISONS['count_non_ascii_chars']
    report['has_non_ascii_chars'] = comparison(count_not_ascii, k)
    
    (tokens_misspelled, tokens_total) = sample.count_misspelled_words(tokens=tokens)
    if DEBUG:
        print(f'tokens_misspelled:  {tokens_misspelled}, tokens_total:  {tokens_total}')
    (comparison, k) = COMPARISONS['ratio_misspelled_words_to_tot_tokens']
    report['has_perfect_spelling'] = comparison(tokens_misspelled / tokens_total, k)

    std_dev_pgraph_length = sample.std_dev_pgraph_length(txt=txt_visible)
    if DEBUG:
        print(f'std_dev_pgraph_length:  {std_dev_pgraph_length}')
    (comparison, k) = COMPARISONS['std_dev_pgraph_sentences']
    report['has_consistent_pgraph_length'] = comparison(std_dev_pgraph_length, k)

    std_dev_sentence_length = sample.std_dev_sentence_length(txt=txt_visible)
    if DEBUG:
        print(f'std_dev_sentence_length:  {std_dev_sentence_length}')
    (comparison, k) = COMPARISONS['std_dev_sentence_words']
    report['has_consistent_sentence_length'] = comparison(std_dev_sentence_length, k)

    sentiment = sample.sentiment_intensity(txt=txt_visible)
    if DEBUG:
        print(f'sentiment[neg]:  {sentiment["neg"]}, sentiment[pos]:  {sentiment["pos"]}')
    (comparison, k) = COMPARISONS['negative_sentiment']
    has_intense_neg_sentiment = comparison(sentiment['neg'], k)
    (comparison, k) = COMPARISONS['positive_sentiment']
    has_intense_pos_sentiment = comparison(sentiment['pos'], k)
    report['has_intense_sentiment'] = has_intense_neg_sentiment or has_intense_pos_sentiment

    count_HAL_speak = sample.count_HAL_speak(HAL_vocab, tokens)
    if DEBUG:
        print(f'count_HAL_speak:  {count_HAL_speak}')
    (comparison, k) = COMPARISONS['has_HAL_speak']
    has_HAL_speak = comparison(count_HAL_speak, k)
    report['has_HAL_speak'] = has_HAL_speak

    pos = sample.part_of_speech_tagging(tokens=tokens)
    collection = sample.pos_in_sequence(pos=pos)
    compression = sample.compression(txt=SPACE.join(collection))
    if DEBUG:
        print(f'compression:  {compression}')
    (comparison, k) = COMPARISONS['is_repetitive']
    is_repetitive = comparison(compression, k)
    report['is_repetitive'] = is_repetitive

    significant_tokens = sample.drop_stop_words(tokens=tokens)
    keywords = sample.keywords(tokens=significant_tokens)
    limit = 7
    count_keywords = [freq for (keyword, freq) in keywords[:limit]]
    keyword_density = sum(count_keywords) / (limit * len(significant_tokens))
    if DEBUG:
        print(f'keyword_density:  {keyword_density}, keywords:  {keywords[:limit]}')
    (comparison, k) = COMPARISONS['has_keyword_stuffing']
    has_keyword_stuffing = comparison(keyword_density, k)
    report['has_keyword_stuffing'] = has_keyword_stuffing

    report.generate()
    return result


def entry_point():
    retcd = main_line()
    sys.exit(retcd)


if __name__ == "__main__":
    entry_point()

    
# Fin
