# coding: utf8
# NOTE: no modules imported here should import `decimal` (otherwise
#   `use_cdecimal` might become problematic for them)

import os
import sys
import inspect
import six
from copy import deepcopy

import math
import time
import functools
from itertools import chain, repeat, islice
import traceback
import re
import unicodedata


__all__ = (
    'bubble',
    'window',
    'dotdict',
    'SmartDict',
    'DebugPlug', 'repr_call',
    'fxrange', 'frange', 'dxrange', 'drange',
    'dict_fget',
    'dict_fsetdefault',
    'interp',
    'edi', 'InterpolationEvaluationException',
    'split_list',
    'use_cdecimal',
    'use_exc_ipdb',
    'use_exc_log',
    'use_colorer',
    'obj2dict',
    'mk_logging_property',
    'sign',
    'try_parse',
    'human_sort_key',
    'reversed_blocks',
    'reversed_lines',
    'lazystr',
    'list_uniq',
    'o_repr',
    'chunks',
    'chunks_g',
    # 'runlib',
    # 'lzmah',
    # 'lzcat',
    # 'psql',
)


def bubble(*args, **kwargs):
    """ Prettified super():
    Call `super(ThisClass, this_instance).this_method(...)`.
    Not super-performant but quite prettifying ("Performance is 5 times
      worse than super() call").
    src:
    http://stackoverflow.com/questions/2706623/super-in-python-2-x-without-args/2706703#2706703
    """
    def find_class_by_code_object(back_self, method_name, code):
        for cls in inspect.getmro(type(back_self)):
            if method_name in cls.__dict__:
                method_fun = getattr(cls, method_name)
                if method_fun.im_func.func_code is code:
                    return cls

    frame = inspect.currentframe().f_back
    back_self = frame.f_locals['self']
    method_name = frame.f_code.co_name

    for _ in xrange(5):
        code = frame.f_code
        cls = find_class_by_code_object(back_self, method_name, code)
        if cls:
            super_ = super(cls, back_self)
            return getattr(super_, method_name)(*args, **kwargs)
        try:
            frame = frame.f_back
        except Exception:
            return


# Iterate over a 'window' of adjacent elements
# http://stackoverflow.com/questions/6998245/iterate-over-a-window-of-adjacent-elements-in-python
def window(seq, size=2, fill=0, fill_left=False, fill_right=False):
    """ Returns a sliding window (of width n) over data from the iterable:
      s -> (s0,s1,...s[n-1]), (s1,s2,...,sn), ...
    """
    ssize = size - 1
    it = chain(
        repeat(fill, ssize * fill_left),
        iter(seq),
        repeat(fill, ssize * fill_right))
    result = tuple(islice(it, size))
    if len(result) == size:  # `<=` if okay to return seq if len(seq) < size
        yield result
    for elem in it:
        result = result[1:] + (elem,)
        yield result


class dotdict(dict):
    """ A simple dict subclass with items also available over attributes """

    def __getattr__(self, name):
        try:
            return self[name]
        except KeyError:
            # A little less confusing way:
            _et, _ev, _tb = sys.exc_info()
            six.reraise(AttributeError, _ev, _tb)

    def __setattr__(self, name, value):
        self[name] = value


# Compat alias
SmartDict = dotdict


def repr_call(ar, kwa):
    """ A helper function for pretty-printing a function call arguments """
    r = ', '.join("%r" % (v,) for v in ar)
    if kwa:
        r += ', ' + ', '.join('%s=%r' % (k, v) for k, v in kwa.iteritems())
    return r


dbgs = {}   # global object for easier later access of dumped `__call__`s


def DebugPlug(name, mklogger=None):
    """ Create and return a recursive duck-object for plugging in
    place of other objects for debug purposes.

    :param name: name for tracking the object and its (child) attributes.

    :param mklogger: a function of `name` (str) that returns a new callable(msg:
      str) used for logging.  See code for an example.
    """
    # The construction with mkClass is for removing the need of
    #   `__getattr__`ing the name and logger.
    import logging

    def mklogger_default(name):
        logger = logging.getLogger(name)
        return logger.debug

    if mklogger is None:
        mklogger = mklogger_default

    log = mklogger(name)

    class DebugPlugInternal(object):
        """ An actual internal class of the DebugPlug """

        def __call__(self, *ar, **kwa):
            log("called with (%s)" % repr_call(ar, kwa))
            global dbgs  ## Just to note it
            dbgs.setdefault(name, {})
            dbgs[name]['__call__'] = (ar, kwa)

        def __getattr__(self, attname):
            namef = "%s.%s" % (name, attname)
            # Recursive!
            dpchild = DebugPlug(name=namef, mklogger=mklogger)
            # setattr(self, attname, dpchild)
            object.__setattr__(self, attname, dpchild)
            return dpchild

        def __setattr__(self, attname, value):
            log("setattr: %s = %r" % (attname, value))
            global dbgs
            dbgs.setdefault(name, {})
            dbgs[name][attname] = value
            return object.__setattr__(self, attname, value)

    return DebugPlugInternal()


# ######
# `range`-like things
# ######


def fxrange(start, end=None, inc=None):
    """ The xrange function for float """
    assert inc != 0, "inc should not be zero"
    if end is None:
        end = start
        start = 0.0
    if inc is None:
        inc = 1.0
    i = 0  # to prevent error accumulation
    while True:
        nextv = start + i * inc
        if (inc > 0 and nextv >= end or
                inc < 0 and nextv <= end):
            break
        yield nextv
        i += 1


def frange(start, end=None, inc=None):
    """ list(fxrange) """
    return list(fxrange(start, end, inc))


def dxrange(start, end=None, inc=None, include_end=False):
    """ The xrange function for Decimal """
    # Imported here mostly because of use_cdecimal in this module
    from decimal import Decimal
    assert inc != 0, "inc should not be zero"
    if end is None:
        end = start
        start = 0
    if inc is None:
        inc = 1
    inc = Decimal(inc)
    start = Decimal(start)
    end = Decimal(end)
    nextv = start
    while True:
        if ((inc > 0) and (not include_end and nextv == end or nextv > end) or
                (inc < 0) and (not include_end and nextv == end or nextv < end)):
            break
        yield nextv
        nextv += inc


def drange(*ar, **kwa):
    """ list(dxrange) """
    return list(dxrange(*ar, **kwa))


def date_xrange(start, end, inc=None, include_end=False, precise=False):
    """ The xrange function for datetime.

    NOTE: the semantics of 'start' and 'end' are different here: with
    end=None an infinite generator is returned.

    :param precise: do more calculations but potentially produce more
        precise results (especially for precise `inc`).

    >>> import datetime
    >>> dt = datetime.datetime
    >>> dta = dt(2011, 11, 11)
    >>> dtb = dt(2011, 11, 14)
    >>> dsl = lambda dtl: [dt.strftime('%Y-%m-%d') for dt in dtl]
    >>> dtsl = lambda dtl: [dt.isoformat() for dt in dtl]
    >>> dsl(date_xrange(dta, dt(2011, 11, 13)))
    ['2011-11-11', '2011-11-12']
    >>> dsl(date_xrange(dta, dt(2011, 11, 13), include_end=True))
    ['2011-11-11', '2011-11-12', '2011-11-13']
    >>> dsl(date_xrange(dta, dtb, inc=2))
    ['2011-11-11', '2011-11-13']
    >>> dtsl(date_xrange(dta, dtb, inc=1111.11111111111 / 86400, precise=True))[-1]
    '2011-11-13T23:54:48.888889'
    >>> dtsl(date_xrange(dta, dtb, inc=1111.11111111111 / 86400))[-1]
    '2011-11-13T23:54:48.888863'
    """
    import datetime
    if inc is None:
        inc = 1  # default: 1 day

    if not isinstance(inc, datetime.timedelta):
        # NOTE: days, by default
        inc_days = inc
        inc = datetime.timedelta(inc)
    else:
        inc_days = inc.total_seconds() / 86400  # py2.7 required

    assert inc_days  # should be nonzero

    is_forward = (inc_days > 0)

    idx = 0
    current = start
    while True:
        if end is not None:
            if include_end:
                to_break = current > end if is_forward else current < end
            else:
                to_break = current >= end if is_forward else current <= end
            if to_break:
                break
        yield current
        if precise:
            idx += 1
            current = start + datetime.timedelta(inc_days * idx)
        else:
            current = current + inc


def date_range(*ar, **kwa):
    """ list(date_xrange) """
    return list(date_xrange(*ar, **kwa))


def date_add_months(sourcedate, months=1):
    """ Add months to date; can cap the day to the maximal value for
    the month """
    import calendar
    month = sourcedate.month - 1 + months
    year = sourcedate.year + month // 12
    month = month % 12 + 1
    day = min(sourcedate.day, calendar.monthrange(year, month)[1])
    # return datetime.date(year, month, day)
    # On the other hand, we can keep the type and the time:
    return sourcedate.replace(year=year, month=month, day=day)


def date_months_xrange(start, end, inc=1, include_end=False):
    """ date_range with delta measured in months.

    Similar semantics to date_xrange.
    Only accepts integer `inc` values.

    >>> import datetime
    >>> dt = datetime.datetime
    >>> dsl = lambda dtl: [dt.strftime('%Y-%m-%d') for dt in dtl]
    >>> dsl(date_months_xrange(dt(2011, 10, 31), dt(2012, 1, 1)))
    ['2011-10-31', '2011-11-30', '2011-12-31']
    """
    assert isinstance(inc, int)
    inc = int(inc)
    assert inc  # should be nonzero
    is_forward = inc > 0
    current = start
    idx = 0
    while True:
        if end is not None:
            if include_end:
                to_break = current > end if is_forward else current < end
            else:
                to_break = current >= end if is_forward else current <= end
            if to_break:
                break
        yield current
        idx += 1
        current = date_add_months(start, months=inc * idx)


def date_months_range(*ar, **kwa):
    """ list(date_months_xrange) """
    return list(date_months_xrange(*ar, **kwa))


# ###### dict-lazies

def dict_fget(D, k, d):
    """ dict_get(D, k, d) -> D[k] if k in D, else d().
      - a lazy-evaluated dict.get.
      (d is mandatory but can be None). """
    if k in D:
        return D[k]
    return d() if d is not None else d


def dict_fsetdefault(D, k, d):
    """ dict_fsetdefault(D, k, d) -> dict_fget(D, k, d), also set D[k]=d() if k not in D.
      - a lazy-evaluated dict.setdefault.
      (d is mandatory but can be None).  """
    # Can be `D[k] = dict_fget(D, k, d); return D[k]`, but let's micro-optimize.
    # NOTE: not going over 'keyerror' for the defaultdict or alike classes.
    if k in D:
        return D[k]
    v = d() if d is not None else d
    D[k] = v
    return v


# ###### String interpolation

# http://rightfootin.blogspot.com/2007/02/string-interpolation-in-python.html
def interp(string, _regexp=r'(#\{([^}]*)\})'):
    """ Inline string interpolation.
    >>> var1 = 213; ff = lambda v: v**2
    >>> interp("var1 is #{var1}")
    'var1 is 213'
    >>> interp("var1 is #{ff(var1)}; also #{ff(12)}")
    'var1 is 45369; also 144'
    """
    fframe = sys._getframe(1)
    flocals = fframe.f_locals
    fglobals = fframe.f_globals
    items = re.findall(_regexp, string)
    item_to_str = {}
    # Do eval and replacement separately and replacement in one regex
    # go to avoid interpolating already interpolated values.
    for item_outer, item in items:
        item_to_str[item] = str(eval(item, fglobals, flocals))
    string = re.sub(_regexp, lambda match: item_to_str[match.group(2)], string)
    return string


# Yet another string-interpolation helper


class InterpolationEvaluationException(KeyError):
    pass


class edi(dict):  # "expression_dictionary"...
    """ Yet another string interpolation helper.

    >>> var1 = 313; f = lambda x: x*2
    >>> print "1 is %(var1)5d, f1 is %(f(var1))d, f is %(f)r, 1/2 is %(float(var1)/2)5.3f." % edi()  #doctest: +ELLIPSIS
    1 is   313, f1 is 626, f is <function <lambda> at 0x...>, 1/2 is 156.500.

    """
    # No idea for what sake this is subclassed from dictionary, actually. A
    # neat extra, perhaps.

    globals = {}

    def __init__(self, d=None):
        if d is None:  # Grab parent's locals forcible
            self.locals = sys._getframe(1).f_locals
            self.globals = sys._getframe(1).f_globals
            d = self.locals
        super(edi, self).__init__(d)

    def __getitem__(self, key):
        try:
            return dict.__getitem__(self, key)
        except KeyError:
            try:
                return eval(key, self.globals, self)
            except Exception, e:
                raise InterpolationEvaluationException(key, e)


# ###### ...

def split_list(lst, cond):
    """ Split list items into two into (matching, non_matching) by
      `cond(item)` callable """
    res1, res2 = [], []
    for i in lst:
        if cond(i):
            res1.append(i)
        else:
            res2.append(i)
    return res1, res2


# ###### Monkey-patching of various things:

def use_cdecimal():
    """ Do a hack-in replacement of `decimal` with `cdecimal`.
    Should be done before importing other modules.  """
    import decimal  # maybe not needed
    import cdecimal
    sys.modules['decimal'] = cdecimal


def use_exc_ipdb():
    """ Set unhandled exception handler to automatically start ipdb """
    import pyaux.exc_ipdb as exc_ipdb
    exc_ipdb.init()


def use_exc_log():
    """ Set unhandled exception handler to verbosely log the exception """
    import pyaux.exc_log as exc_log
    exc_log.init()


def use_colorer():
    """ Wrap logging's StreamHandler.emit to add colors to the logged
      messages based on log level """
    # TODO: make a ColorerHandlerMixin version
    import pyaux.Colorer as Colorer
    Colorer.init()


# ...

def obj2dict(o, add_type=False, add_instance=False, do_lists=True,
             dict_class=dotdict):
    """" Recursive o -> o.__dict__ """
    kwa = dict(add_type=add_type, add_instance=add_instance,
               do_lists=do_lists, dict_class=dict_class)

    if hasattr(o, '__dict__'):
        res = dict_class()
        for k, v in o.__dict__.iteritems():
            res[k] = obj2dict(v, **kwa)
        if add_type:
            res['__class__'] = o.__class__
        if add_instance:
            res['__instance__'] = o
        return res
    # Recurse through other types too:
    # NOTE: There might be subclasses of these that would not be
    # processed here.
    elif isinstance(o, dict):
        return dict_class((k, obj2dict(v, **kwa)) for k, v in o.iteritems())
    elif isinstance(o, list):
        return [obj2dict(v, **kwa) for v in o]

    return o  # something else - return as-is.


def mk_logging_property(actual_name, logger_name='_log'):
    """ Creates a property that logs the value and the caller in the
    setter, using logger under `self`'s logger_name, and stores the value
    under actual_name on `self` """

    def do_get(self):
        return getattr(self, actual_name)

    def do_set(self, val):
        tb = traceback.extract_stack(limit=2)[0]
        # # or:
        # next((r.f_code.co_filename, r.f_lineno, r.f_code.co_name) for r in (sys._getframe(1),))
        # # that is,
        # r = sys._getframe(1)
        # co = r.f_code
        # co.co_filename, r.f_lineno, co.co_name
        setattr(self, actual_name, val)
        getattr(self, logger_name).debug(
            "%s set to %r from %s:%d, in %s",
            actual_name, val, tb[0], tb[1], tb[2])

    return property(do_get, do_set)


def sign(v):
    """ Sign of value: `return cmp(v, 0)` """
    return cmp(v, 0)


# ######  "Human" sorting, advanced
# partially from quodlibet/quodlibet/util/__init__.py
# partially from comix/src/filehandler.py


def try_parse(v, fn=int):
    """ 'try parse' (with fn) """
    try:
        return fn(v)
    except Exception, e:
        return v


# Note: not localized (i.e. always as dot for decimal separator)
_re_alphanum_f = re.compile(r'[0-9]+(?:\.[0-9]+)?|[^0-9]+')


def _split_numeric_f(s):
    return [try_parse(v, fn=float) for v in _re_alphanum_f.findall(s)]


# Or to avoid interpreting numbers as float:
_re_alphanum_int = re.compile(r'\d+|\D+')


def _split_numeric(s):
    return [try_parse(v, fn=int) for v in _re_alphanum_int.findall(s)]


# Primary function:
def human_sort_key(s, normalize=unicodedata.normalize, floats=True):
    """ Sorting key for 'human' sorting """
    if not isinstance(s, unicode):
        s = s.decode("utf-8")
    s = normalize("NFD", s.lower())
    split_fn = _split_numeric_f if floats else _split_numeric
    return s and split_fn(s)


# ###### Reading files backwards
# http://stackoverflow.com/a/260433/62821

def reversed_blocks(fileobj, blocksize=4096):
    """ Generate blocks of file's contents in reverse order.  """
    fileobj.seek(0, os.SEEK_END)
    here = fileobj.tell()
    while 0 < here:
        delta = min(blocksize, here)
        fileobj.seek(here - delta, os.SEEK_SET)
        yield fileobj.read(delta)
        here -= delta


def reversed_lines(fileobj):
    """ Generate the lines of file in reverse order.  """
    tail = []           # Tail of the line whose head is not yet read.
    for block in reversed_blocks(fileobj):
        # A line is a list of strings to avoid quadratic concatenation.
        # (And trying to avoid 1-element lists would complicate the code.)
        linelists = [[line] for line in block.splitlines()]
        linelists[-1].extend(tail)
        for linelist in reversed(linelists[1:]):
            yield ''.join(linelist)
        tail = linelists[0]
    if tail:
        yield ''.join(tail)


# ####### ...

class ThrottledCall(object):
    """ Decorator for throttling calls to some functions (e.g. logging).
    Defined as class for various custom attributes and methods.
    Attributes:
      `handle_skip`: function(self, *ar, **kwa) to call when a call is
        skipped. (default: return None)
    Methods: `call_something`
    """

    # _last_call_time = None
    _call_time_throttle = None
    _call_cnt = 0  # (kept accurate; but can become ineffectively large)
    _call_cnt_throttle = 0  # next _call_cnt to call at
    _call_val = object()  # (some unique value at start)

    def __init__(self, fn=None, sec_limit=None, cnt_limit=None):
        """ `fn`: function to call (can be customized later).
        `sec_limit`: skip call if less than `sec_limit` seconds since the
          last call
        `cnt_limit`: call only once each `cnt_limit` calls
        """
        self.fn = fn
        # # mimickry, v2
        # self.__call__ = wraps(fn)(self.__call__)
        self.sec_limit = sec_limit
        self.cnt_limit = cnt_limit
        doc = "%s (throttled)" % (fn.__doc__,)
        self.__call__.__func__.__doc__ = self.__doc__ = doc
        self.handle_skip = lambda self, *ar, **kwa: None

    def __call__(self, *ar, **kwa):
        return self.call_something(self.fn, *ar, **kwa)

    def call_something(self, fn, *ar, **kwa):
        """ Call some (other) function with the same throttling """
        now = time.time()
        # do_call = True
        # # NOTE: `throttle_cnt(throttle_sec(fn))` is emulated if both
        # # are set.
        if self.cnt_limit is not None:
            self._call_cnt += 1
            if self._call_cnt >= self._call_cnt_throttle:
                self._call_cnt_throttle += self.cnt_limit
            else:
                return self.handle_skip(self, *ar, **kwa)
        if self.sec_limit is not None:
            if now > self._call_time_throttle:
                self._call_time_throttle = now + self.sec_limit
            else:
                return self.handle_skip(self, *ar, **kwa)
        res = fn(*ar, **kwa)
        return res

    def call_value(self, val, fn, *ar, **kwa):
        """ Call if the value hasn't changed (applying the other
        throttling parameters as well). Contains undocumented feature
        """
        if self._call_val != val:
            self._call_val = val
            if fn is None:
                fn = self.fn
            return self.call_something(fn, *ar, **kwa)

    def __repr__(self):
        return "<throttled_call(%r)>" % (self.fn,)

    # # Optional: mimickry
    # def __repr__(self):
    #     return repr(self.fn)
    # def __getattr__(self, v):
    #     return getattr(self.fn, v)


@functools.wraps(ThrottledCall)
def throttled_call(fn=None, *ar, **kwa):
    """ Wraps the supplied function with ThrottledCall (or generates a
    wrapper with the supplied parameters). """
    if fn is not None:
        if callable(fn):
            # mimickry, v3
            return functools.wraps(fn)(ThrottledCall(fn, *ar, **kwa))
        else:  # supplied some arguments as positional?
            # XX: make a warning?
            ar = (fn,) + ar
    return lambda fn: functools.wraps(fn)(ThrottledCall(fn, *ar, **kwa))


class lazystr(object):
    """ A simple class for lazy-computed processing into string,
      e.g. for use in logging.
    Example: `log(13, "stuff is %r",
      lazystr(lambda: ', '.join(stuff)))`
    Note: no caching.
    """

    def __init__(self, fn):
        self.fn = fn

    def __str__(self):
        return str(self.fn())

    def __repr__(self):
        return repr(self.fn())


def uniq_g(lst, key=lambda v: v):
    """ Get unique elements of an iterable preserving its order and optionally
    determining uniqueness by hash of a key """
    known = set()
    for v in lst:
        k = key(v)
        if k not in known:
            yield v
            known.add(k)
    # ...


def uniq(lst, key=lambda v: v):
    """ RTFS """
    return list(uniq_g(lst, key=key))


list_uniq = uniq_g


# Helper for o_repr that displays '???'
class ReprObj(object):
    """ A class for inserting specific text in __repr__ outputs.  """

    def __init__(self, txt):
        self.txt = txt

    def __repr__(self):
        return self.txt


# _err_obj = type('ErrObj', (object,), dict(__repr__=lambda self: '???'))()
_err_obj = ReprObj('???')


# It is similar to using self.__dict__ in repr() but works over dir()
def o_repr_g(
        o, _colors=False, _colors256=False, _colorvs=None, _method=2,
        _private=False, _callable=False):
    """ Represent (most of) data on a python object in readable
    way. Useful default for a __repr__.
    WARN: does not handle recursive structures; use carefully.  """

    # TODO: handle recursive structures (similar to dict.__repr__)
    # TODO: process base types (dict / list / ...) in a special way

    def _color(x16, x256=None):
        if not _colors:
            return ''
        if x16 is None and x256 is None:
            return '\x1b[00m'
        if _colors256 and x256:
            return '\x1b[38;5;' + str(x256) + 'm'
        return '\x1b[0' + str(x16) + 'm'

    _colorvs_add = _colorvs
    _colorvs = dict(
        base=('1;37', '230'),  # Base (punctuation)  # white / pink-white
        clsn=('1;36', '123'),  # Class name  # bright cyan / cyan-white
        attr=('0;32', '120'),  # Attribute name  # dark-green / green-white
        func=('0;34', '75'),  # Function name  # blue / blue-cyan
        val=('0;37', '252'),  # Value data  # light-gray / light-light-gray
        clear=(None, None),
    )
    _colorvs.update(_colorvs_add or {})

    def _colorv(n):
        return _color(*_colorvs[n])

    yield _colorv("base")
    yield '<'
    yield _colorv("clsn")
    yield str(o.__class__.__name__)
    yield _colorv("base")
    yield '('

    if _method == 3:
        # V3: check type for properties
        o_type = type(o)

    first = True

    for n in sorted(dir(o)):

        # skip 'private' stuff
        if n.startswith('_') and not _private:
            continue

        if first:
            first = False
        else:
            yield _colorv("base")
            yield ', '

        keytype = "attr"
        has_val = True

        # V2: try but fail
        if _method == 2:
            try:
                v = getattr(o, n)
                if callable(v):  # skip functions (... and other callables)
                    if not _callable:
                        has_val = False
                    keytype = "func"
            except Exception:
                v = _err_obj

        if _method == 3:
            # V3: check type for properties
            v_m = getattr(o_type, n, None)
            if v_m is not None and isinstance(v_m, property):
                continue  # skip properties
            v = getattr(o, n)

            # skip functions (... and other callables)
            if callable(v):
                if not _callable:
                    has_val = False
                keytype = "func"

        yield _colorv(keytype)
        yield str(n)  # NOTE: some cases (e.g. functions) will remain just names

        if has_val:
            try:
                v = repr(v)
            except Exception:
                v = _err_obj
            yield _colorv("base")
            yield '='
            yield _colorv("val")
            yield v

    yield _colorv("base")
    yield ')>'
    yield _colorv("clear")


def o_repr(o, **kwa):
    return ''.join(o_repr_g(o, **kwa))


class OReprMixin(object):
    def __repr__(self):
        return o_repr(self)


def stdin_lines(strip_newlines=True):
    """ Iterate over stdin lines in a 'line-buffered' way """
    while True:
        try:
            l = sys.stdin.readline()
        except KeyboardInterrupt:
            return
        if not l:
            break
        # This might not be the case if the stream terminates with a non-newline at the end.
        if strip_newlines and l[-1] == '\n':
            l = l[:-1]
        yield l


def stdout_lines(gen):
    """ Send lines from a generator / iterable to stdout in a line-buffered way. """
    for l in gen:
        sys.stdout.write("%s\n" % (l,))
        sys.stdout.flush()


def dict_merge(target, source, instancecheck=None, dictclass=dict,
               del_obj=object(), _copy=True, inplace=False):
    """ do update() on 'dict of dicts of di...' structure recursively.
    Also, see sources for details.
    NOTE: does not keep target's specific tree structure (forces source's)
    :param del_obj: allows for deletion of keys if the key in the `source` is set to this.

    >>> data = {}
    >>> data = dict_merge(data, {'open_folders': {'my_folder_a': False}})
    >>> data
    {'open_folders': {'my_folder_a': False}}
    >>> data = dict_merge(data, {'open_folders': {'my_folder_b': True}})
    >>> data
    {'open_folders': {'my_folder_b': True, 'my_folder_a': False}}
    >>> _del = object()
    >>> data = dict_merge(data, {'open_folders': {'my_folder_b': _del}}, del_obj=_del)
    >>> data
    {'open_folders': {'my_folder_a': False}}
    """
    if instancecheck is None:  # funhorrible ducktypings
        # instancecheck = lambda iv: isinstance(iv, dict)

        def instancecheck_default(iv):
            return hasattr(iv, 'iteritems')

        instancecheck = instancecheck_default

    # Recursive parameters shorthand
    kwa = dict(instancecheck=instancecheck, dictclass=dictclass, del_obj=del_obj)

    if _copy or not inplace:  # 'either is default'
        target = deepcopy(target)

    for k, v in source.iteritems():
        if v is del_obj:
            target.pop(k, None)
        elif instancecheck(v):  # (v -> source -> iteritems())
            # NOTE: if target[k] wasn't a dict - it will be, now.
            target[k] = dict_merge(
                dict_fget(target, k, dictclass), v, **kwa)
        else:  # nowhere to recurse into - just replace
            # NOTE: if target[k] was a dict - it won't be, anymore.
            target[k] = v

    return target


def _sqrt(var):
    """ Duck-compatible sqrt(), needed to support classes like Decimal.

    Approximately the same as in the python3's `statistics`. """
    try:
        return var.sqrt()
    except AttributeError:
        return math.sqrt(var)


class IterStat(object):
    """ Iterative single-pass computing of mean and variance.

    Error is on the rate of 1e-08 for 1e6 values in the range of
    0..1e6, both for mean and for stddev. """
    # http://www.johndcook.com/standard_deviation.html

    def __init__(self, vals=None, start=0):
        self.start = start
        self.old_mean = None
        self.mean = self.stdx = start
        self.cnt = 0

        if vals:
            for val in vals:
                self.send(val)

    def send(self, val):
        self.cnt += 1
        if self.cnt == 1:
            self.mean = val
        else:
            self.mean = self.mean + (val - self.mean) / float(self.cnt)
            self.stdx = self.stdx + (val - self.old_mean) * (val - self.mean)
        self.old_mean = self.mean

    @property
    def variance(self):
        if self.cnt <= 1:
            return self.start
        return self.stdx / (self.cnt)

    @property
    def std(self):
        return _sqrt(self.variance)


def chunks(lst, size):
    """ Yield successive chunks from lst. No padding.  """
    for i in xrange(0, len(lst), size):
        yield lst[i:i + size]


def chunks_g(iterable, size):
    """ Same as 'chunks' but works on any iterable.

    Converts the chunks to tuples for simplicity.
    """
    # http://stackoverflow.com/a/8991553
    it = iter(iterable)
    if size <= 0:
        yield it
        return
    while True:
        chunk = tuple(islice(it, size))
        if not chunk:
            return
        yield chunk


def group(lst, cls=dict):
    """ RTFS.

    Similar to dict(MultiValueDict(lst).lists())

    >>> group([(1, 1), (2, 2), (1, 3)])
    {1: [1, 3], 2: [2]}
    """
    res = cls()
    for key, val in lst:
        try:
            group_list = res[key]
        except KeyError:
            group_list = []
            res[key] = group_list
        group_list.append(val)
    return res


def mangle_dict(input_dict, include=None, exclude=None, add=None, _return_list=False):
    res = []
    include = set(include) if include is not None else None
    exclude = set(exclude) if exclude is not None else None
    if include:
        assert not exclude
    # TODO?: make a clever one-liner of all this stuff.
    for key, val in input_dict.iteritems():
        if include is not None:
            if key in include:
                res.append((key, val))
        elif exclude is not None:
            if key not in exclude:
                res.append((key, val))
        else:  # no includes/excludes
            res.append((key, val))
    # ... functional-style `update`.
    if add is not None:
        if isinstance(add, dict):
            add = add.iteritems()
        for key, val in add:
            res.append((key, val))
    if _return_list:
        return res
    return dict(res)


def colorize(text, fmt, outfmt='term', lexer_kwa=None, formatter_kwa=None, **kwa):
    """ Convenience method for running pygments """
    from pygments import highlight, lexers, formatters
    _colorize_lexers = dict(
        yaml='YamlLexer', diff='DiffLexer',
        py='PythonLexer', py2='PythonLexer', py3='Python3Lexer',
    )
    _colorize_formatters = dict(
        term='TerminalFormatter', term256='Terminal256Formatter',
        html='HtmlFormatter',
    )
    fmt = _colorize_lexers.get(fmt, fmt)
    outfmt = _colorize_formatters.get(outfmt, outfmt)

    lexer_cls = getattr(lexers, fmt)
    formatter_cls = getattr(formatters, outfmt)
    return highlight(
        text, lexer_cls(**(lexer_kwa or {})),
        formatter_cls(**(formatter_kwa or {})))


def colorize_yaml(text, **kwa):
    """ Attempt to colorize the yaml text using pygments (for console
    output) """
    return colorize(text, 'yaml', **kwa)


def colorize_diff(text, **kwa):
    """ Attempt to colorize the [unified] diff text using pygments
    (for console output) """
    return colorize(text, 'diff', **kwa)
