test_runner_from_cli API documentation

# Module test_runner_from_cli

Show source.

    import unittest
    import sys
    import subprocess
    import inspect
    import json
    import csv
    import sofine.runner as runner
    import sofine.lib.utils.utils as utils

    class RunnerFromCliTestCase(unittest.TestCase):

        def test_runner_main(self):
            # NOTE: Piped (and single non-piped) command lines need to enclose the
            #  entire set of piped calls in quotes, which here are single quotes
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
            # NOTE: This call failed with this error message: "sys.excepthook is missing.
            #  lost sys.stderr" until adding stderr arg to Popen() call
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 2)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

        def test_runner_main_namespaced(self):
            cmd_get_namespaced_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
            cmd_get_namespaced_data += " | "
            cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 2)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

            for k in out.keys():
                self.assertTrue(len(out[k]))
                for attr in out[k]:
                    for k,v in attr.iteritems():
                        self.assertTrue(k is not None and k.startswith('mock::ystockquotelib_mock::') and v is not None)

        def test_runner_main_formats(self):
            cmd_get_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-d format_json --SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            json_out = proc.stdout.read().strip()

            cmd_get_data = "echo 'AAPL|MSFT|'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-d format_csv --SF-s ystockquotelib_mock --SF-g mock'"
            proc2 = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            csv_out = proc2.stdout.read()

            # Added bonus, also test the deserialize methods in the plugins
            #  as well as that the data returned is identical in CSV and JSON data format
            json_plugin = utils.load_plugin_module('format_json')
            json_out = json_plugin.deserialize(json_out)
            csv_plugin = utils.load_plugin_module('format_csv')
            csv_out = csv_plugin.deserialize(csv_out)

            # NOTE: These two data formats are NOT isomorphic, because CSV has no notion of
            #  types other thans sting, it is a pure text format. JSON is a data type with the
            #  typed literals, with the same type support as JavaScript. As a result, roundtripping
            #  data through CSV gives you back a Python dict with all keys and all values as
            #  strings, whatever type they were previously
            # So we need to stringify all the keys and values in the deserialized JSON and then
            #  comapre the CSV deserialization. This tells us the outputs are as "equivalent" as
            #  we can assert, given the loss of types in the CSV conversion
            json_keys = set([str(k) for k in json_out.keys()])
            json_attr_keys = set([str(k) for k in utils.get_attr_keys(json_out)])
            json_attr_vals = set([str(v) for v in utils.get_attr_values(json_out)])
            csv_keys = set(csv_out.keys())
            csv_attr_keys = set(utils.get_attr_keys(csv_out))
            csv_attr_vals = set(utils.get_attr_values(csv_out))

            self.assertTrue(json_keys == csv_keys and
                            json_attr_keys == csv_attr_keys and
                            json_attr_vals == csv_attr_vals)

        def test_runner_main_pipe(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}'".format(path_2)
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 4)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR']))

        def test_runner_main_pipeline(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 6)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
            # Assert that the final output has values from ystockquotelib_mock for
            #  keys added by file_source
            for k in out.keys():
                attr_keys = [attr.keys()[0] for attr in out[k]]
                self.assertTrue(len(attr_keys))
                for attr in out[k]:
                    self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

        def test_runner_main_namespaced_pipeline(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 6)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
            # Assert that the final output has values from ystockquotelib_mock for
            #  keys added by file_source
            for k in out.keys():
                attr_keys = [attr.keys()[0] for attr in out[k]]
                self.assertTrue(len(attr_keys))
                for attr in out[k]:
                    self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

        def test_runner_main_get_schema(self):
            cmd_get_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_schema'"
            proc = subprocess.Popen(cmd_get_schema, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            schema_out = proc.stdout.read()
            schema_out = json.loads(schema_out)
            schema_attr_keys = set(schema_out['schema'])

            cmd_get_data = "echo '{\"TWTR\" : []}'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            data_out = proc.stdout.read()
            data_out = json.loads(data_out)
            data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])

            self.assertTrue(schema_attr_keys == data_attr_keys)

        def test_runner_main_get_namespaced_schema(self):
            cmd_get_namespaced_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_schema'"
            proc = subprocess.Popen(cmd_get_namespaced_schema , shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            schema_out = proc.stdout.read()
            schema_out = json.loads(schema_out)
            schema_attr_keys = set(schema_out['schema'])

            cmd_get_namespaced_data = "echo '{\"TWTR\" : []}'"
            cmd_get_namespaced_data += " | "
            cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            data_out = proc.stdout.read()
            data_out = json.loads(data_out)
            data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])

            self.assertTrue(schema_attr_keys == data_attr_keys)

        def test_runner_main_adds_keys(self):
            cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a adds_keys'"
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(out['adds_keys'])

        def test_runner_main_parse_args(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a parse_args -p {0}'".format(path)
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(out['is_valid'] and out['parsed_args'] == [path])

        def test_runner_main_piped_input(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd = "echo '{\"TWTR\" : {}}'"
            cmd += " | "
            cmd += "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(set(out.keys()) == set(['TWTR', 'AAPL', 'MSFT']))

    if __name__ == '__main__':
        unittest.main()

## Classes

class RunnerFromCliTestCase

Show source.

    class RunnerFromCliTestCase(unittest.TestCase):

        def test_runner_main(self):
            # NOTE: Piped (and single non-piped) command lines need to enclose the
            #  entire set of piped calls in quotes, which here are single quotes
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
            # NOTE: This call failed with this error message: "sys.excepthook is missing.
            #  lost sys.stderr" until adding stderr arg to Popen() call
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 2)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

        def test_runner_main_namespaced(self):
            cmd_get_namespaced_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
            cmd_get_namespaced_data += " | "
            cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 2)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

            for k in out.keys():
                self.assertTrue(len(out[k]))
                for attr in out[k]:
                    for k,v in attr.iteritems():
                        self.assertTrue(k is not None and k.startswith('mock::ystockquotelib_mock::') and v is not None)

        def test_runner_main_formats(self):
            cmd_get_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-d format_json --SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            json_out = proc.stdout.read().strip()

            cmd_get_data = "echo 'AAPL|MSFT|'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-d format_csv --SF-s ystockquotelib_mock --SF-g mock'"
            proc2 = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            csv_out = proc2.stdout.read()

            # Added bonus, also test the deserialize methods in the plugins
            #  as well as that the data returned is identical in CSV and JSON data format
            json_plugin = utils.load_plugin_module('format_json')
            json_out = json_plugin.deserialize(json_out)
            csv_plugin = utils.load_plugin_module('format_csv')
            csv_out = csv_plugin.deserialize(csv_out)

            # NOTE: These two data formats are NOT isomorphic, because CSV has no notion of
            #  types other thans sting, it is a pure text format. JSON is a data type with the
            #  typed literals, with the same type support as JavaScript. As a result, roundtripping
            #  data through CSV gives you back a Python dict with all keys and all values as
            #  strings, whatever type they were previously
            # So we need to stringify all the keys and values in the deserialized JSON and then
            #  comapre the CSV deserialization. This tells us the outputs are as "equivalent" as
            #  we can assert, given the loss of types in the CSV conversion
            json_keys = set([str(k) for k in json_out.keys()])
            json_attr_keys = set([str(k) for k in utils.get_attr_keys(json_out)])
            json_attr_vals = set([str(v) for v in utils.get_attr_values(json_out)])
            csv_keys = set(csv_out.keys())
            csv_attr_keys = set(utils.get_attr_keys(csv_out))
            csv_attr_vals = set(utils.get_attr_values(csv_out))

            self.assertTrue(json_keys == csv_keys and
                            json_attr_keys == csv_attr_keys and
                            json_attr_vals == csv_attr_vals)

        def test_runner_main_pipe(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}'".format(path_2)
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 4)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR']))

        def test_runner_main_pipeline(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 6)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
            # Assert that the final output has values from ystockquotelib_mock for
            #  keys added by file_source
            for k in out.keys():
                attr_keys = [attr.keys()[0] for attr in out[k]]
                self.assertTrue(len(attr_keys))
                for attr in out[k]:
                    self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

        def test_runner_main_namespaced_pipeline(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
            path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
            cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
            cmd_get_data += " | "
            cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(len(out.keys()) == 6)
            self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
            # Assert that the final output has values from ystockquotelib_mock for
            #  keys added by file_source
            for k in out.keys():
                attr_keys = [attr.keys()[0] for attr in out[k]]
                self.assertTrue(len(attr_keys))
                for attr in out[k]:
                    self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

        def test_runner_main_get_schema(self):
            cmd_get_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_schema'"
            proc = subprocess.Popen(cmd_get_schema, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            schema_out = proc.stdout.read()
            schema_out = json.loads(schema_out)
            schema_attr_keys = set(schema_out['schema'])

            cmd_get_data = "echo '{\"TWTR\" : []}'"
            cmd_get_data += " | "
            cmd_get_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock'"
            proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            data_out = proc.stdout.read()
            data_out = json.loads(data_out)
            data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])

            self.assertTrue(schema_attr_keys == data_attr_keys)

        def test_runner_main_get_namespaced_schema(self):
            cmd_get_namespaced_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_schema'"
            proc = subprocess.Popen(cmd_get_namespaced_schema , shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            schema_out = proc.stdout.read()
            schema_out = json.loads(schema_out)
            schema_attr_keys = set(schema_out['schema'])

            cmd_get_namespaced_data = "echo '{\"TWTR\" : []}'"
            cmd_get_namespaced_data += " | "
            cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
            proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

            data_out = proc.stdout.read()
            data_out = json.loads(data_out)
            data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])

            self.assertTrue(schema_attr_keys == data_attr_keys)

        def test_runner_main_adds_keys(self):
            cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a adds_keys'"
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(out['adds_keys'])

        def test_runner_main_parse_args(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a parse_args -p {0}'".format(path)
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(out['is_valid'] and out['parsed_args'] == [path])

        def test_runner_main_piped_input(self):
            path = './sofine/tests/fixtures/file_source_test_data.txt'
            cmd = "echo '{\"TWTR\" : {}}'"
            cmd += " | "
            cmd += "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
            proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            out = proc.stdout.read()
            out = json.loads(out)
            self.assertTrue(set(out.keys()) == set(['TWTR', 'AAPL', 'MSFT']))

#### Methods

def test_runner_main(

self)

Show source.

    def test_runner_main(self):
        # NOTE: Piped (and single non-piped) command lines need to enclose the
        #  entire set of piped calls in quotes, which here are single quotes
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
        # NOTE: This call failed with this error message: "sys.excepthook is missing.
        #  lost sys.stderr" until adding stderr arg to Popen() call
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(len(out.keys()) == 2)
        self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

def test_runner_main_adds_keys(

self)

Show source.

    def test_runner_main_adds_keys(self):
        cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a adds_keys'"
        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(out['adds_keys'])

def test_runner_main_formats(

self)

Show source.

    def test_runner_main_formats(self):
        cmd_get_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
        cmd_get_data += " | "
        cmd_get_data += "python ./sofine/runner.py '--SF-d format_json --SF-s ystockquotelib_mock --SF-g mock'"
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        json_out = proc.stdout.read().strip()

        cmd_get_data = "echo 'AAPL|MSFT|'"
        cmd_get_data += " | "
        cmd_get_data += "python ./sofine/runner.py '--SF-d format_csv --SF-s ystockquotelib_mock --SF-g mock'"
        proc2 = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        csv_out = proc2.stdout.read()
        # Added bonus, also test the deserialize methods in the plugins
        #  as well as that the data returned is identical in CSV and JSON data format
        json_plugin = utils.load_plugin_module('format_json')
        json_out = json_plugin.deserialize(json_out)
        csv_plugin = utils.load_plugin_module('format_csv')
        csv_out = csv_plugin.deserialize(csv_out)
        # NOTE: These two data formats are NOT isomorphic, because CSV has no notion of
        #  types other thans sting, it is a pure text format. JSON is a data type with the
        #  typed literals, with the same type support as JavaScript. As a result, roundtripping
        #  data through CSV gives you back a Python dict with all keys and all values as
        #  strings, whatever type they were previously
        # So we need to stringify all the keys and values in the deserialized JSON and then
        #  comapre the CSV deserialization. This tells us the outputs are as "equivalent" as
        #  we can assert, given the loss of types in the CSV conversion
        json_keys = set([str(k) for k in json_out.keys()])
        json_attr_keys = set([str(k) for k in utils.get_attr_keys(json_out)])
        json_attr_vals = set([str(v) for v in utils.get_attr_values(json_out)])
        csv_keys = set(csv_out.keys())
        csv_attr_keys = set(utils.get_attr_keys(csv_out))
        csv_attr_vals = set(utils.get_attr_values(csv_out))
        self.assertTrue(json_keys == csv_keys and
                        json_attr_keys == csv_attr_keys and
                        json_attr_vals == csv_attr_vals)

def test_runner_main_get_namespaced_schema(

self)

Show source.

    def test_runner_main_get_namespaced_schema(self):
        cmd_get_namespaced_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_schema'"
        proc = subprocess.Popen(cmd_get_namespaced_schema , shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        schema_out = proc.stdout.read()
        schema_out = json.loads(schema_out)
        schema_attr_keys = set(schema_out['schema'])
        cmd_get_namespaced_data = "echo '{\"TWTR\" : []}'"
        cmd_get_namespaced_data += " | "
        cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
        proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        data_out = proc.stdout.read()
        data_out = json.loads(data_out)
        data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])

        self.assertTrue(schema_attr_keys == data_attr_keys)

def test_runner_main_get_schema(

self)

Show source.

    def test_runner_main_get_schema(self):
        cmd_get_schema = "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_schema'"
        proc = subprocess.Popen(cmd_get_schema, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        schema_out = proc.stdout.read()
        schema_out = json.loads(schema_out)
        schema_attr_keys = set(schema_out['schema'])
        cmd_get_data = "echo '{\"TWTR\" : []}'"
        cmd_get_data += " | "
        cmd_get_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock'"
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        data_out = proc.stdout.read()
        data_out = json.loads(data_out)
        data_attr_keys = set([attr.keys()[0] for attr in data_out['TWTR']])
        self.assertTrue(schema_attr_keys == data_attr_keys)

def test_runner_main_namespaced(

self)

Show source.

    def test_runner_main_namespaced(self):
        cmd_get_namespaced_data = "echo '{\"AAPL\" : [], \"MSFT\" : []}'"
        cmd_get_namespaced_data += " | "
        cmd_get_namespaced_data += "python ./sofine/runner.py '--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
        proc = subprocess.Popen(cmd_get_namespaced_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(len(out.keys()) == 2)
        self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT']))

        for k in out.keys():
            self.assertTrue(len(out[k]))
            for attr in out[k]:
                for k,v in attr.iteritems():
                    self.assertTrue(k is not None and k.startswith('mock::ystockquotelib_mock::') and v is not None)

def test_runner_main_namespaced_pipeline(

self)

Show source.

    def test_runner_main_namespaced_pipeline(self):
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
        path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
        cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock --SF-a get_namespaced_data'"
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(len(out.keys()) == 6)
        self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
        # Assert that the final output has values from ystockquotelib_mock for
        #  keys added by file_source
        for k in out.keys():
            attr_keys = [attr.keys()[0] for attr in out[k]]
            self.assertTrue(len(attr_keys))
            for attr in out[k]:
                self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

def test_runner_main_parse_args(

self)

Show source.

    def test_runner_main_parse_args(self):
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        cmd = "python ./sofine/runner.py '--SF-s file_source --SF-g standard --SF-a parse_args -p {0}'".format(path)
        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(out['is_valid'] and out['parsed_args'] == [path])

def test_runner_main_pipe(

self)

Show source.

    def test_runner_main_pipe(self):
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
        cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s file_source --SF-g standard -p {0}'".format(path_2)
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(len(out.keys()) == 4)
        self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR']))

def test_runner_main_piped_input(

self)

Show source.

    def test_runner_main_piped_input(self):
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        cmd = "echo '{\"TWTR\" : {}}'"
        cmd += " | "
        cmd += "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}'".format(path)
        proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(set(out.keys()) == set(['TWTR', 'AAPL', 'MSFT']))

def test_runner_main_pipeline(

self)

Show source.

    def test_runner_main_pipeline(self):
        path = './sofine/tests/fixtures/file_source_test_data.txt'
        path_2 = './sofine/tests/fixtures/file_source_test_data_2.txt'
        path_3 = './sofine/tests/fixtures/file_source_test_data_3.txt'
        cmd_get_data = "python ./sofine/runner.py '--SF-s file_source --SF-g standard -p {0}".format(path)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_2)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s file_source --SF-g standard -p {0}".format(path_3)
        cmd_get_data += " | "
        cmd_get_data += "--SF-s ystockquotelib_mock --SF-g mock'"
        proc = subprocess.Popen(cmd_get_data, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out = proc.stdout.read()
        out = json.loads(out)
        self.assertTrue(len(out.keys()) == 6)
        self.assertTrue(set(out.keys()) == set(['AAPL', 'MSFT', 'MCO', 'TWTR', 'IBM', 'ORCL']))
        # Assert that the final output has values from ystockquotelib_mock for
        #  keys added by file_source
        for k in out.keys():
            attr_keys = [attr.keys()[0] for attr in out[k]]
            self.assertTrue(len(attr_keys))
            for attr in out[k]:
                self.assertTrue(attr.keys()[0] is not None and attr.values()[0] is not None)

Documentation generated by `pdoc` 0.2.4. pdoc is in the public domain with the UNLICENSE.