Coverage for curator/logtools.py: 85%

1"""Logging tools"""

2 import sys

3 import json

4 import logging

5 import time

6 from curator.exceptions import LoggingException

7

8 def de_dot(dot_string, msg):

9 """Turn message and dotted string into a nested dictionary"""

10 arr = dot_string.split('.')

11 arr.append(msg)

12 retval = None

13 for idx in range(len(arr), 1, -1):

14 if not retval:

15 try:

16 retval = {arr[idx-2]: arr[idx-1]}

17 except Exception as err:

18 raise LoggingException(err)

19 else:

20 try:

21 new_d = {arr[idx-2]: retval}

22 retval = new_d

23 except Exception as err:

24 raise LoggingException(err)

25 return retval

26

27 def deepmerge(source, destination):

28 """Merge deeply nested dictionary structures"""

29 for key, value in source.items():

30 if isinstance(value, dict):

31 node = destination.setdefault(key, {})

32 deepmerge(value, node)

33 else:

34 destination[key] = value

35 return destination

36 class LogstashFormatter(logging.Formatter):

37 """Logstash formatting (JSON)"""

38 # The LogRecord attributes we want to carry over to the Logstash message,

39 # mapped to the corresponding output key.

40 WANTED_ATTRS = {

41 'levelname': 'loglevel',

42 'funcName': 'function',

43 'lineno': 'linenum',

44 'message': 'message',

45 'name': 'name'

46 }

47

48 def format(self, record):

49 self.converter = time.gmtime

50 timestamp = '%s.%03dZ' % (

51 self.formatTime(record, datefmt='%Y-%m-%dT%H:%M:%S'), record.msecs)

52 result = {'@timestamp': timestamp}

53 available = record.__dict__

54 # This is cleverness because 'message' is NOT a member key of ``record.__dict__``

55 # the ``getMessage()`` method is effectively ``msg % args`` (actual keys)

56 # By manually adding 'message' to ``available``, it simplifies the code

57 available['message'] = record.getMessage()

58 for attribute in set(self.WANTED_ATTRS).intersection(available):

59 result = deepmerge(

60 de_dot(self.WANTED_ATTRS[attribute], getattr(record, attribute)), result

61 )

62 # The following is mostly for the ecs format. You can't have 2x 'message' keys in

63 # WANTED_ATTRS, so we set the value to 'log.original' in ecs, and this code block

64 # guarantees it still appears as 'message' too.

65 if 'message' not in result.items():

66 result['message'] = available['message']

67 return json.dumps(result, sort_keys=True)

68

69 class ECSFormatter(LogstashFormatter):

70 """Elastic Common Schema formatting (ECS)"""

71 # Overload LogstashFormatter attribute

72 WANTED_ATTRS = {

73 'levelname': 'log.level',

74 'funcName': 'log.origin.function',

75 'lineno': 'log.origin.file.line',

76 'message': 'log.original',

77 'name': 'log.logger'

78 }

79

80 class Whitelist(logging.Filter):

81 """How to whitelist logs"""

82 def __init__(self, *whitelist):

83 self.whitelist = [logging.Filter(name) for name in whitelist]

84

85 def filter(self, record):

86 return any(f.filter(record) for f in self.whitelist)

87

88 class Blacklist(Whitelist):

89 """Blacklist monkey-patch of Whitelist"""

90 def filter(self, record):

91 return not Whitelist.filter(self, record)

92

93 class LogInfo(object):

94 """Logging Class"""

95 def __init__(self, cfg):

96 cfg['loglevel'] = 'INFO' if not 'loglevel' in cfg else cfg['loglevel']

97 cfg['logfile'] = None if not 'logfile' in cfg else cfg['logfile']

98 cfg['logformat'] = 'default' if not 'logformat' in cfg else cfg['logformat']

99 self.numeric_log_level = getattr(logging, cfg['loglevel'].upper(), None)

100 self.format_string = '%(asctime)s %(levelname)-9s %(message)s'

101 if not isinstance(self.numeric_log_level, int):

102 raise ValueError('Invalid log level: {0}'.format(cfg['loglevel']))

103

104 self.handler = logging.StreamHandler(

105 open(cfg['logfile'], 'a') if cfg['logfile'] else sys.stdout

106 )

107

108 if self.numeric_log_level == 10: # DEBUG

109 self.format_string = (

110 '%(asctime)s %(levelname)-9s %(name)22s '

111 '%(funcName)22s:%(lineno)-4d %(message)s'

112 )

113

114 if cfg['logformat'] == 'json' or cfg['logformat'] == 'logstash':

115 self.handler.setFormatter(LogstashFormatter())

116 elif cfg['logformat'] == 'ecs':

117 self.handler.setFormatter(ECSFormatter())

118 else:

119 self.handler.setFormatter(logging.Formatter(self.format_string))