# Copyright (c) 2009-2010, Robert Escriva, Rensselaer Polytechnic Institute
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 
#     * Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of this project nor the names of its contributors may
#       be used to endorse or promote products derived from this software
#       without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.


'''Parse files generated by the collection process.
'''


import subprocess
import sys


try:
    import simplejson
except ImportError:
    import warnings
    WARN = 'Falling back to ``json`` module.\n' + \
           '    You will see an order see a 17x performance *loss* ' + \
           'without ``simplejson``'
    warnings.warn(WARN)
    import json as simplejson


def statusiterator(filename):
    r'''A generator which takes care of decompressing and splitting statuses.

    Note that just a simple splitting across newlines fails as we've observed
    Twitter use every style of newline ('\r', '\n', '\r\n').
    '''
    if filename.endswith('.json.bz2'):
        pipe = subprocess.Popen(['bunzip2', '--stdout', filename],
                stdout=subprocess.PIPE)
        stream = pipe.stdout
    elif filename.endswith('.json.gz'):
        pipe = subprocess.Popen(['gunzip', '--stdout', filename],
                stdout=subprocess.PIPE)
        stream = pipe.stdout
    elif filename.endswith('.json'):
        pipe = None
        stream = open(filename)
    else:
        raise RuntimeError('Provide a filename of the right form')

    for line in stream:
        # This hocus pocus is necessary because twitter doesn't use consistent
        # line endings.
        if line.endswith('\r\n'):
            line = line[:-2]
        if '\r' in line:
            lines = line.split('\r')
            for trueline in lines:
                if trueline:
                    yield trueline
        else:
            if line:
                yield line
    stream.close()
    if pipe is not None:
        pipe.wait()


def parse(filename, action):
    '''Parse a filename and call ``action`` with each valid status.

    ``filename`` is the name of a file matching '\*.json', '\*.json.gz' or
    '\*.json.bz2'.  Decompression is automatically performed using the commands
    ``gunzip`` and ``bzip2`` respectively.  Each parsed status will be passed as
    an argument to the ``action`` callable.
    '''
    for line in statusiterator(filename):
        try:
            d = simplejson.loads(line)
        except ValueError:
            print >> sys.stderr, 'error parsing:', line
        else:
            action(d)
