#!/usr/bin/env python
#
# Copyright (c) 2011-2013, Shopkick Inc.
# All rights reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ---
# Author: John Egan <john@shopkick.com>

import copy
import datetime
import email
import json
import time
import unittest
import StringIO

from flawless.lib.data_structures.stubs import PersistentDictionaryStub
import flawless.lib.config
from flawless.server import api
from flawless.server.service import *
import flawless.server.server



class BaseTestCase(unittest.TestCase):

  def setUp(self):
    super(BaseTestCase, self).setUp()
    self.popen_stub = POpenStub()
    self.popen_stub.stdout = StringIO.StringIO(
      "75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
      "author John Egan\n"
      "author-mail <wishbone@shopkick.com>\n"
      "author-time %d\n"
      "author-tz -0800\n"
      "committer John Egan\n"
      "committer-mail <repo_master@shopkick.com>\n"
      "committer-time 1356245776\n"
      "committer-tz -0800\n"
      "summary Add more robust support for string keys\n"
      "previous 3491c7b8e298ec81dc7583163a118e7c2250999f safe_access.py\n"
      "filename safe_access.py\n"
      "               ex: a.b.[myvar] where myvar is passed in as a kwarg\n"
      % int(time.mktime(datetime.datetime(2017, 7, 30).timetuple()))
    )

    self._set_stub_time(datetime.datetime(2020, 1, 1))
    self.smtp_stub = SMTPClientStub()
    self.file_open_stub = OpenFileStub()

    self.watchers = [
      "TEST COMMENT",
      {
        "email": "wilfred@shopkick.com",
        "filepath": "tools/furminator.py",
        "watch_all_errors": True,
      },
      {
        "email": "lassie@shopkick.com",
        "filepath": "tools/furminator.py",
        "watch_all_errors": True,
      },
      "TEST COMMENT",
      {
        "email": "wishbone@shopkick.com",
        "filepath": "lib/no/such/path/for/testing.py",
        "watch_all_errors": True,
      },
    ]

    self.third_party_whitelist = [
      "TEST COMMENT",
      ThirdPartyWhitelistEntry('facebook.py', 'post_treat_to_facebook',
                     'urllib.urlencode(args), post_data)'),
      "TEST COMMENT",
      ThirdPartyWhitelistEntry(
          'SQLAlchemy-0.5.6-py2.6.egg/sqlalchemy/pool.py',
          'do_get',
          'raise exc.TimeoutError("QueuePool limit of size %d overflow %d ',
      ),
    ]

    self.known_errors = [
      KnownError(
        'lib/doghouse/authentication.py',
        'check_auth',
        'raise errors.BadAuthenticationError("Something smells off...")',
        max_alert_threshold=0,
      ),
      "TEST COMMENT",
      KnownError(
        filename='coreservices/waterbowl/rewards/water.py',
        function_name='make_external_api_request',
        code_fragment='raise api.WaterbowlError(error_code='
                      'api.WaterbowlErrorCode.OUT_OF_WATER, message=str(e))',
        email_recipients=["wilfred@shopkick.com", "snoopy@shopkick.com"],
        email_header = 'NOTE: This error typically does not require dev team involvement.',
        alert_every_n_occurences=1,
      ),
    ]

    self.building_blocks = [
      "TEST COMMENT",
      BuildingBlock('apps/shopkick/doghouse/lib/base.py',
                     '_get_request_param',
                     'raise errors.BadRequestError("Missing param %s" % name)'),
    ]

    self.file_open_stub.set_file('../config/building_blocks', dump_json(self.building_blocks))

    self.file_open_stub.set_file('../config/third_party_whitelist',
                                 dump_json(self.third_party_whitelist))
    self.file_open_stub.set_file('../config/known_errors', dump_json(self.known_errors))
    self.file_open_stub.set_file('../config/email_remapping', dump_json([]))
    self.file_open_stub.set_file('../config/watched_files', dump_json(self.watchers))

    self.saved_config = copy.deepcopy(flawless.lib.config.get().__dict__)
    self.test_config = flawless.lib.config.get()
    self.test_config.__dict__ = dict((o.name, o.default) for o in flawless.lib.config.OPTIONS)
    self.test_config.repo_dir = "/tmp"
    self.test_config.report_only_after_minimum_date = "2010-01-01"
    self.test_config.report_error_threshold = 1
    self.test_config.only_blame_filepaths_matching = [
      r"^coreservices(?!.*/thrift/).*$",
      r"lib/.*",
      r"tools/.*",
    ]
    self.test_config.report_runtime_package_directory_name = "site-packages"
    self.test_config.config_dir_path = "../config"

    self.handler = FlawlessService(
      open_process_func=self.popen_stub,
      persistent_dict_cls=PersistentDictionaryStub,
      smtp_client_cls=self.smtp_stub,
      time_func=lambda: self.stub_time,
      open_file_func=self.file_open_stub,
      thread_cls=ThreadStub,
    )

  def tearDown(self):
    super(BaseTestCase, self).tearDown()
    flawless.lib.config.get().__dict__ = self.saved_config

  def _set_stub_time(self, dt):
    self.stub_time = int(time.mktime(dt.timetuple()))

  def assertDictEquals(self, expected, actual):
    if expected == actual:
      return

    bad_keys = set(expected.keys()) ^ set(actual.keys())
    if bad_keys:
      keys_str = [str(key) for key in (set(expected.keys()) - set(actual.keys()))]
      errstr = "Keys not in second dict: %s" % "\n".join(keys_str)
      keys_str = [str(key) for key in (set(actual.keys()) - set(expected.keys()))]
      errstr += "\nKeys not in first dict:  %s" % "\n".join(keys_str)
      raise AssertionError("Missing/Extraneous keys:\n%s" % errstr)
    for key in expected.keys():
      if expected[key] != actual[key]:
        raise AssertionError("Value mismatch for key %s:\n%s\n%s" %
                             (key, str(expected[key]), str(actual[key])))

  def assertEmailEquals(self, expected, actual):
    self.assertEquals(expected["from_address"], actual["from_address"])
    self.assertEquals(set(expected["to_addresses"]), set(actual["to_addresses"]))
    parsed_email = email.message_from_string(actual["body"])
    self.assertEquals(expected["from_address"], parsed_email["From"])
    self.assertEquals(set(expected["to_addresses"]), set(parsed_email["To"].split(", ")))
    self.assertEquals(expected["subject"], parsed_email["Subject"])
    self.assertTrue(expected["body"] in parsed_email.get_payload(decode=True))


class RecordErrorTestCase(BaseTestCase):

  def setUp(self):
    super(RecordErrorTestCase, self).setUp()

  def test_records_error(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
    )

    self.handler.record_error(req.dumps())
    self.assertDictEquals({api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00",
                                    is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)
    self.assertEqual(["git", "--git-dir=/tmp/.git", "--work-tree=/tmp", "blame",
                      "-p", "/tmp/coreservices/service.py", "-L", "7,+1"],
                     self.popen_stub.last_args)
    self.assertEmailEquals(dict(to_addresses=["wishbone@shopkick.com"],
                          from_address="error_report@example.com",
                          subject="Error on localhost in coreservices/service.py",
                          body="email text",
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)

  def test_records_error_with_thrift_in_file_name(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/coreservices/thrift_file.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
    )

    self.handler.record_error(req.dumps())
    self.assertDictEquals({api.ErrorKey("coreservices/thrift_file.py", 7, "serve", "..."):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00",
                                    is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)

  def test_ignores_error_in_thrift_directory(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x"),
                   api.StackLine("/site-packages/coreservices/thrift/file.py", 7, "serve", "...")],
        exception_message="email text",
        hostname="localhost",
    )

    self.handler.record_error(req.dumps())
    self.assertDictEquals({}, self.handler.errors_seen.dict)

  def test_doesnt_report_errors_under_threshold(self):
    self.test_config.report_error_threshold = 2
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
    )

    self.handler.record_error(req.dumps())
    self.assertDictEquals({api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2017-07-30 00:00:00", False, "2020-01-01 00:00:00",
                                    is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)
    self.assertEquals(None, self.smtp_stub.last_args)

  def test_uses_threshold_specified_in_request(self):
    self.test_config.report_error_threshold = 2
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
        error_threshold=1,
    )

    self.handler.record_error(req.dumps())

    self.assertEmailEquals(dict(to_addresses=["wishbone@shopkick.com"],
                          from_address="error_report@example.com",
                          cc_address=None,
                          bcc_address=None,
                          subject="Error on localhost in coreservices/service.py",
                          body="email text",
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)

  def test_always_alerts_on_red_alert_errors(self):
    self.test_config.report_error_threshold = 3
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine('/site-packages/coreservices/service/utils.py',
                                   9, "check_water_levels", '% (waterbowl_id, min_required_level))'),
                   api.StackLine("/site-packages/coreservices/waterbowl/rewards/water.py",
                                   5,
                                   "make_external_api_request",
                                   "raise api.WaterbowlError(error_code=api.WaterbowlErrorCode."
                                   "OUT_OF_WATER, message=str(e))"),
        ],
        exception_message="email text",
        hostname="localhost",
    )

    self.handler.record_error(req.dumps())

    # The 2 red alert recipients plus the developer responsible
    self.assertEmailEquals(dict(to_addresses=["wilfred@shopkick.com", "wishbone@shopkick.com",
                                              "snoopy@shopkick.com"],
                          from_address="error_report@example.com",
                          cc_address=None,
                          bcc_address=None,
                          subject="Error on localhost in coreservices/waterbowl/rewards/water.py",
                          body="email text",
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)

  def test_email_includes_watchers(self):
    # Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
    # a path that Yen is watching
    self.test_config.report_error_threshold = 2
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/tools/furminator.py", 7, "fubar", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
        error_threshold=1,
    )

    self.handler.record_error(req.dumps())
    self.assertEmailEquals(dict(to_addresses=["wilfred@shopkick.com", "wishbone@shopkick.com", "lassie@shopkick.com"],
                          from_address="error_report@example.com",
                          cc_address=None,
                          bcc_address=None,
                          subject="Error on localhost in tools/furminator.py",
                          body="email text",
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)

  def test_email_includes_extra_information(self):
    # Traceback includes a path that has extra_information tagged on it
    self.test_config.report_error_threshold = 2
    req = api.RecordErrorRequest(
        traceback=[
            api.StackLine(
                "/site-packages/coreservices/waterbowl/rewards/water.py",
                5,
                "make_external_api_request",
                "raise api.WaterbowlError(error_code=api.WaterbowlErrorCode."
                "OUT_OF_WATER, message=str(e))"
            ),
        ],
        exception_message="email text",
        hostname="localhost",
        error_threshold=1,
        additional_info="extra stuff",
    )

    self.handler.record_error(req.dumps())

    self.assertEmailEquals(dict(to_addresses=["snoopy@shopkick.com",
                                      "wilfred@shopkick.com",
                                      "wishbone@shopkick.com"],
                          from_address="error_report@example.com",
                          cc_address=None,
                          bcc_address=None,
                          subject="Error on localhost in coreservices/waterbowl/rewards/water.py",
                          body='NOTE: This error typically does not require dev team involvement.',
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)
    body = email.message_from_string(self.smtp_stub.last_args["body"]).get_payload(decode=True)
    self.assertTrue("email text" in body)
    self.assertTrue("extra stuff" in body)

  def test_removes_duplicate_emails(self):
    # Almost the same setup as test_uses_threshold_specified_in_request except the traceback includes
    # a path that John is watching. Since he's also the developer who committed the buggy code, he is
    # the one and only email recipient.
    self.test_config.report_error_threshold = 2
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/lib/no/such/path/for/testing.py", 7, "fubar", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
        error_threshold=1,
    )

    self.handler.record_error(req.dumps())

    self.assertEmailEquals(dict(to_addresses=["wishbone@shopkick.com"],
                          from_address="error_report@example.com",
                          cc_address=None,
                          bcc_address=None,
                          subject="Error on localhost in lib/no/such/path/for/testing.py",
                          body="email text",
                          smtp_server_host_port=None),
                     self.smtp_stub.last_args)



  def test_doesnt_email_on_errors_before_cutoff_date(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
    )
    self.popen_stub.stdout = StringIO.StringIO(
      "75563df6e9d1efe44b48f6643fde9ebbd822b0c5 25 25 1\n"
      "author John Egan\n"
      "author-mail <wishbone@shopkick.com>\n"
      "author-time %d\n"
      "author-tz -0800\n"
      % int(time.mktime(datetime.datetime(2009, 7, 30).timetuple()))
    )

    self.handler.record_error(req.dumps())
    self.assertDictEquals({api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2009-07-30 00:00:00", False,
                                    "2020-01-01 00:00:00", is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)
    self.assertEqual(None, self.smtp_stub.last_args)

  def test_records_error_only_once(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/thirdparty/3rdparty_lib.py", 9, "call", "x")],
        exception_message="email text",
        hostname="localhost",
    )
    self.handler.record_error(req.dumps())
    self._set_stub_time(datetime.datetime(2020, 1, 2))
    self.handler.record_error(req.dumps())

    self.assertDictEquals({api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
                      api.ErrorInfo(2, "wishbone@shopkick.com", "2017-07-30 00:00:00", True, "2020-01-02 00:00:00",
                                    is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)
    self.assertEqual(1, len(self.smtp_stub.args_list))

  def test_does_not_email_for_whitelisted_errors(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/lib/doghouse/authentication.py", 7, "check_auth",
                                   'raise errors.BadAuthenticationError("Something smells off...")')],
        exception_message="email text",
        hostname="localhost",
    )
    self.handler.record_error(req.dumps())

    self.assertDictEquals({api.ErrorKey("lib/doghouse/authentication.py", 7, "check_auth",
                                    'raise errors.BadAuthenticationError("Something smells off...")'):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2017-07-30 00:00:00", False, "2020-01-01 00:00:00",
                                    is_known_error=True, last_error_data=req)},
                     self.handler.errors_seen.dict)
    self.assertEquals(None, self.smtp_stub.last_args)

  def test_ignores_third_party_whitelisted_errors(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/SQLAlchemy-0.5.6-py2.6.egg/sqlalchemy/pool.py",
                                   7,
                                   "do_get",
                                   'raise exc.TimeoutError("QueuePool limit of size %d overflow %d '
                                   'reached, connection timed out, timeout %d" % (self.size(),'
                                   'self.overflow(), self._timeout))')],
        exception_message="email text",
        hostname="localhost",
    )
    self.handler.record_error(req.dumps())
    self.assertDictEquals({}, self.handler.errors_seen.dict)

  def test_ignores_third_party_whitelisted_errors_for_facebook(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/facebook.py",
                                   7,
                                   "post_treat_to_facebook",
                                   'urllib.urlencode(args), post_data)')],
        exception_message="email text",
        hostname="localhost",
    )
    self.handler.record_error(req.dumps())
    self.assertDictEquals({}, self.handler.errors_seen.dict)

  def test_traces_up_stack_trace_for_errors_originating_from_building_blocks(self):
    req = api.RecordErrorRequest(
        traceback=[api.StackLine("/site-packages/lib/test.py", 5, "test_func", "code"),
                   api.StackLine("/site-packages/coreservices/service.py", 7, "serve", "..."),
                   api.StackLine("/site-packages/apps/shopkick/doghouse/lib/base.py", 9,
                                       "_get_request_param",
                                       'raise errors.BadRequestError("Missing param %s" % name)')],
        exception_message="email text",
        hostname="localhost",
    )
    self.handler.record_error(req.dumps())

    self.assertDictEquals({api.ErrorKey("coreservices/service.py", 7, "serve", "..."):
                      api.ErrorInfo(1, "wishbone@shopkick.com", "2017-07-30 00:00:00", True, "2020-01-01 00:00:00",
                                    is_known_error=False, last_error_data=req)},
                     self.handler.errors_seen.dict)



############################## Stubs ##############################
class LogStub(object):

  def __init__(self):
    self.last_args = None
    self.args_list = []

  def info(self, args):
    self.last_args = args
    self.args_list.append(args)


class POpenStub(object):

  def __init__(self):
    self.last_args = None
    self.stdout = StringIO.StringIO()
    self.stderr = StringIO.StringIO()

  def __call__(self, args, **kwargs):
    self.last_args = args
    return self


class SMTPClientStub(object):
  def __init__(self):
    self.args_list = []
    self.last_args = None
    self.host = None
    self.port = None

  def __call__(self, host, port):
    self.host = host
    self.port = port
    return self

  def sendmail(self, from_address, to_addresses, body):
    self.last_args = dict((k,v) for k, v in locals().items() if k != 'self')
    self.args_list.append(self.last_args)

  def quit(self):
    pass

  def login(self, user, password):
    pass


class OpenFileStub(object):
  def __init__(self):
    self.files = dict()
    self.current_file = None

  def set_file(self, filename, contents):
    self.files[filename] = StringIO.StringIO(contents)

  def __enter__(self, *args, **kwargs):
    return self.files[self.current_file]

  def __exit__(self, type, value, traceback):
    pass

  def __call__(self, filename, *args, **kwargs):
    self.current_file = filename
    return self


class ThreadStub(object):
  def __init__(self, target=None, args=[]):
    self.target = target
    self.args = args

  def start(self):
    if self.target.__name__ != '_run_background_update_thread':
      self.target(*self.args)


if __name__ == "__main__":
  unittest.main()
