Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/defaults.py: 100%

1"""App Defaults"""

2

3 import typing as t

4 from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required, Schema

5 from es_wait.defaults import EXISTS, HEALTH, ILM, RELOCATE, RESTORE, SNAPSHOT, TASK

6

7 TRACKING_INDEX = 'redactions-tracker'

8

9 CLICK_DRYRUN = {

10 'dry-run': {

11 'help': 'Do not perform any changes.',

12 'is_flag': True,

13 'show_envvar': True,

14 'envvar': 'PII_TOOL_DRY_RUN',

15 }

16}

17

18 CLICK_TRACKING = {

19 'tracking-index': {

20 'help': 'Name for the tracking index.',

21 'default': TRACKING_INDEX,

22 'show_default': True,

23 'show_envvar': True,

24 'envvar': 'PII_TOOL_TRACKING_INDEX',

25 }

26}

27

28 PHASES: t.Sequence = ['hot', 'warm', 'cold', 'frozen', 'delete']

29

30 PAUSE_DEFAULT: str = '9.0'

31 PAUSE_ENVVAR: str = 'PII_TOOL_PAUSE'

32 TIMEOUT_DEFAULT: str = '7200.0'

33 TIMEOUT_ENVVAR: str = 'PII_TOOL_TIMEOUT'

34

35 TIMINGS: dict = {

36 'exists': {

37 'pause': {

38 'testing': 0.3,

39 'default': EXISTS.get('pause'),

40 },

41 'timeout': {

42 'testing': 10.0,

43 'default': EXISTS.get('timeout'),

44 },

45 },

46 'health': {

47 'pause': {

48 'testing': 0.3,

49 'default': HEALTH.get('pause'),

50 },

51 'timeout': {

52 'testing': 10.0,

53 'default': HEALTH.get('timeout'),

54 },

55 },

56 'ilm': {

57 'pause': {

58 'testing': 1.0,

59 'default': ILM.get('pause'),

60 },

61 'timeout': {

62 'testing': 30.0,

63 'default': ILM.get('timeout'),

64 },

65 },

66 'relocate': {

67 'pause': {

68 'testing': 0.5,

69 'default': RELOCATE.get('pause'),

70 },

71 'timeout': {

72 'testing': 30.0,

73 'default': RELOCATE.get('timeout'),

74 },

75 },

76 'restore': {

77 'pause': {

78 'testing': 0.5,

79 'default': RESTORE.get('pause'),

80 },

81 'timeout': {

82 'testing': 30.0,

83 'default': RESTORE.get('timeout'),

84 },

85 },

86 'snapshot': {

87 'pause': {

88 'testing': 0.5,

89 'default': SNAPSHOT.get('pause'),

90 },

91 'timeout': {

92 'testing': 30.0,

93 'default': SNAPSHOT.get('timeout'),

94 },

95 },

96 'task': {

97 'pause': {

98 'testing': 0.3,

99 'default': TASK.get('pause'),

100 },

101 'timeout': {

102 'testing': 30.0,

103 'default': TASK.get('timeout'),

104 },

105 },

106}

107

108

109 def forcemerge_schema() -> t.Dict[Optional, t.Union[All, Any, Coerce, Range, Required]]:

110 """Define the forcemerge schema"""

111 return {

112 Optional('max_num_segments', default=1): All(

113 Coerce(int), Range(min=1, max=32768)

114 ),

115 # The Boolean() here is a capitalized function, not a class. This code passes

116 # without the need for the passed value because of how voluptuous Schema

117 # validation works.

118 # pylint: disable=no-value-for-parameter

119 Optional('only_expunge_deletes', default=False): Any(

120 bool, All(Any(str), Boolean())

121 ),

122 }

123

124

125 def redactions_schema() -> t.Dict[

126 Optional,

127 t.Dict[

128 t.Union[Required, Optional],

129 t.Union[All, Any, t.Dict, t.Sequence[Any], Optional],

130 ],

131]:

132 """An index pattern to search and redact data from"""

133 merge = forcemerge_schema()

134 return {

135 Optional(Any(str)): {

136 Required('pattern'): Any(str),

137 Required('query'): {Any(str): dict},

138 Required('fields'): [Any(str)],

139 Required('message', default='REDACTED'): Any(str),

140 # The Boolean() here is a capitalized function, not a class. This code

141 # passes without the need for the passed value because of how voluptuous

142 # Schema validation works.

143 # pylint: disable=no-value-for-parameter

144 Optional('delete', default=True): Any(bool, All(Any(str), Boolean())),

145 Required('expected_docs'): All(Coerce(int), Range(min=1, max=32768)),

146 Optional('restore_settings', default=None): Any(dict, None),

147 Optional('forcemerge'): merge,

148 }

149 }

150

151

152 def index_settings() -> t.Dict:

153 """The Elasticsearch index settings for the progress/status tracking index"""

154 return {

155 'index': {

156 'number_of_shards': '1',

157 'auto_expand_replicas': '0-1',

158 }

159 }

160

161

162 def status_mappings() -> t.Dict:

163 """The Elasticsearch index mappings for the progress/status tracking index"""

164 return {

165 'properties': {

166 'job': {'type': 'keyword'},

167 'task': {'type': 'keyword'},

168 'step': {'type': 'keyword'},

169 'join_field': {'type': 'join', 'relations': {'job': 'task'}},

170 'cleanup': {'type': 'keyword'},

171 'completed': {'type': 'boolean'},

172 'end_time': {'type': 'date'},

173 'errors': {'type': 'boolean'},

174 'dry_run': {'type': 'boolean'},

175 'index': {'type': 'keyword'},

176 'logs': {'type': 'text'},

177 'start_time': {'type': 'date'},

178 },

179 'dynamic_templates': [

180 {

181 'configuration': {

182 'path_match': 'config.*',

183 'mapping': {'type': 'keyword', 'index': False},

184 }

185 }

186 ],

187 }

188

189

190 def redaction_schema() -> Schema:

191 """The full voluptuous Schema for a redaction file"""

192 return Schema({Required('redactions'): [redactions_schema()]})