Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/defaults.py: 100%

1"""App Defaults"""

2

3 import typing as t

4 from voluptuous import All, Any, Boolean, Coerce, Optional, Range, Required, Schema

5

6 TRACKING_INDEX = 'redactions-tracker'

7

8 CLICK_DRYRUN = {

9 'dry-run': {

10 'help': 'Do not perform any changes.',

11 'is_flag': True,

12 'show_envvar': True,

13 'envvar': 'PII_TOOL_DRY_RUN',

14 }

15}

16

17 CLICK_TRACKING = {

18 'tracking-index': {

19 'help': 'Name for the tracking index.',

20 'default': TRACKING_INDEX,

21 'show_default': True,

22 'show_envvar': True,

23 'envvar': 'PII_TOOL_TRACKING_INDEX',

24 }

25}

26

27 PHASES: t.Sequence = ['hot', 'warm', 'cold', 'frozen', 'delete']

28

29 PAUSE_DEFAULT: str = '9.0'

30 PAUSE_ENVVAR: str = 'PII_TOOL_PAUSE'

31 TIMEOUT_DEFAULT: str = '7200.0'

32 TIMEOUT_ENVVAR: str = 'PII_TOOL_TIMEOUT'

33

34

35 def forcemerge_schema() -> t.Dict[Optional, t.Union[All, Any, Coerce, Range, Required]]:

36 """Define the forcemerge schema"""

37 return {

38 Optional('max_num_segments', default=1): All(

39 Coerce(int), Range(min=1, max=32768)

40 ),

41 # The Boolean() here is a capitalized function, not a class. This code passes

42 # without the need for the passed value because of how voluptuous Schema

43 # validation works.

44 # pylint: disable=no-value-for-parameter

45 Optional('only_expunge_deletes', default=False): Any(

46 bool, All(Any(str), Boolean())

47 ),

48 }

49

50

51 def redactions_schema() -> t.Dict[

52 Optional,

53 t.Dict[

54 t.Union[Required, Optional],

55 t.Union[All, Any, t.Dict, t.Sequence[Any], Optional],

56 ],

57]:

58 """An index pattern to search and redact data from"""

59 merge = forcemerge_schema()

60 return {

61 Optional(Any(str)): {

62 Required('pattern'): Any(str),

63 Required('query'): {Any(str): dict},

64 Required('fields'): [Any(str)],

65 Required('message', default='REDACTED'): Any(str),

66 # The Boolean() here is a capitalized function, not a class. This code

67 # passes without the need for the passed value because of how voluptuous

68 # Schema validation works.

69 # pylint: disable=no-value-for-parameter

70 Optional('delete', default=True): Any(bool, All(Any(str), Boolean())),

71 Required('expected_docs'): All(Coerce(int), Range(min=1, max=32768)),

72 Optional('restore_settings', default=None): Any(dict, None),

73 Optional('forcemerge'): merge,

74 }

75 }

76

77

78 def index_settings() -> t.Dict:

79 """The Elasticsearch index settings for the progress/status tracking index"""

80 return {

81 'index': {

82 'number_of_shards': '1',

83 'auto_expand_replicas': '0-1',

84 }

85 }

86

87

88 def status_mappings() -> t.Dict:

89 """The Elasticsearch index mappings for the progress/status tracking index"""

90 return {

91 'properties': {

92 'job': {'type': 'keyword'},

93 'task': {'type': 'keyword'},

94 'join_field': {'type': 'join', 'relations': {'job': 'task'}},

95 'cleanup': {'type': 'keyword'},

96 'completed': {'type': 'boolean'},

97 'end_time': {'type': 'date'},

98 'errors': {'type': 'boolean'},

99 'dry_run': {'type': 'boolean'},

100 'index': {'type': 'keyword'},

101 'logs': {'type': 'text'},

102 'start_time': {'type': 'date'},

103 },

104 'dynamic_templates': [

105 {

106 'configuration': {

107 'path_match': 'config.*',

108 'mapping': {'type': 'keyword', 'index': False},

109 }

110 }

111 ],

112 }

113

114

115 def redaction_schema() -> Schema:

116 """The full voluptuous Schema for a redaction file"""

117 return Schema({Required('redactions'): [redactions_schema()]})