Coverage for curator/actions/cold2frozen.py: 65%

1"""Snapshot and Restore action classes"""

2 import logging

3 from curator.helpers.getters import get_alias_actions, get_frozen_prefix, get_tier_preference

4 from curator.helpers.testers import has_lifecycle_name, is_idx_partial, verify_index_list

5 from curator.helpers.utils import report_failure

6 from curator.exceptions import CuratorException, FailedExecution, SearchableSnapshotException

7

8 class Cold2Frozen:

9 """Cold to Frozen Tier Searchable Snapshot Action Class

10

11 For manually migrating snapshots not associated with ILM from the cold tier to the frozen tier.

12 """

13

14 DEFAULTS = {

15 'index_settings': None,

16 'ignore_index_settings': ['index.refresh_interval'],

17 'wait_for_completion': True,

18 }

19 def __init__(self, ilo, **kwargs):

20 """

21 :param ilo: An IndexList Object

22 :param index_settings: (Optional) Settings that should be added to the index when it is

23 mounted. If not set, set the ``_tier_preference`` to the tiers available, coldest

24 first.

25 :param ignore_index_settings: (Optional, array of strings) Names of settings that should

26 be removed from the index when it is mounted.

27 :param wait_for_completion: Wait for completion before returning.

28

29 :type ilo: :py:class:`~.curator.indexlist.IndexList`

30 :type index_settings: dict

31 :type ignore_index_settings: list

32 :type wait_for_completion: bool

33 """

34 self.loggit = logging.getLogger('curator.actions.cold2frozen')

35 verify_index_list(ilo)

36 # Check here and don't bother with the rest of this if there are no

37 # indices in the index list.

38 ilo.empty_list_check()

39

40 #: The :py:class:`~.curator.indexlist.IndexList` object passed from param ``ilo``

41 self.index_list = ilo

42 #: The :py:class:`~.elasticsearch.Elasticsearch` client object derived from

43 #: :py:attr:`index_list`

44 self.client = ilo.client

45 #: Object attribute that contains the :py:func:`~.curator.helpers.utils.to_csv` output of

46 #: the indices in :py:attr:`index_list`.

47 self.indices = ilo

48 #: Object attribute that gets the value of ``index_settings``.

49 self.index_settings = None

50 #: Object attribute that gets the value of ``ignore_index_settings``.

51 self.ignore_index_settings = None

52 #: Object attribute that gets the value of param ``wait_for_completion``.

53 self.wait_for_completion = None

54

55 # Parse the kwargs into attributes

56 self.assign_kwargs(**kwargs)

57

58 def assign_kwargs(self, **kwargs):

59 """

60 Assign the kwargs to the attribute of the same name with the passed value or the default

61 from DEFAULTS

62 """

63 # Handy little loop here only adds kwargs that exist in DEFAULTS, or the default value.

64 # It ignores any non-relevant kwargs

65 for key, value in self.DEFAULTS.items():

66 if key in kwargs:

67 setattr(self, key, kwargs[key])

68 else:

69 setattr(self, key, value)

70

71 def action_generator(self):

72 """Yield a dict for use in :py:meth:`do_action` and :py:meth:`do_dry_run`

73

74 :returns: A generator object containing the settings necessary to migrate indices from cold

75 to frozen

76 :rtype: dict

77 """

78 for idx in self.index_list.indices:

79 idx_settings = self.client.indices.get(index=idx)[idx]['settings']['index']

80 if has_lifecycle_name(idx_settings):

81 self.loggit.critical(

82 'Index %s is associated with an ILM policy and this action will never work on '

83 'an index associated with an ILM policy', idx)

84 raise CuratorException(f'Index {idx} is associated with an ILM policy')

85 if is_idx_partial(idx_settings):

86 self.loggit.critical('Index %s is already in the frozen tier', idx)

87 raise SearchableSnapshotException('Index is already in frozen tier')

88 snap = idx_settings['store']['snapshot']['snapshot_name']

89 snap_idx = idx_settings['store']['snapshot']['index_name']

90 repo = idx_settings['store']['snapshot']['repository_name']

91 aliases = self.client.indices.get(index=idx)[idx]['aliases']

92

93 prefix = get_frozen_prefix(snap_idx, idx)

94 renamed = f'{prefix}{snap_idx}'

95

96 if not self.index_settings:

97 self.index_settings = {

98 "routing": {

99 "allocation": {

100 "include": {

101 "_tier_preference": get_tier_preference(self.client)

102 }

103 }

104 }

105 }

106 yield {

107 'repository': repo, 'snapshot': snap, 'index': snap_idx,

108 'renamed_index': renamed, 'index_settings': self.index_settings,

109 'ignore_index_settings': self.ignore_index_settings,

110 'storage': 'shared_cache', 'wait_for_completion': self.wait_for_completion,

111 'aliases': aliases, 'current_idx': idx

112 }

113

114 def do_dry_run(self):

115 """Log what the output would be, but take no action."""

116 self.loggit.info('DRY-RUN MODE. No changes will be made.')

117 for kwargs in self.action_generator():

118 aliases = kwargs.pop('aliases')

119 current_idx = kwargs.pop('current_idx')

120 msg = (

121 f'DRY-RUN: cold2frozen: from snapshot {kwargs["snapshot"]} in repository '

122 f'{kwargs["repository"]}, mount index {kwargs["index"]} renamed as '

123 f'{kwargs["renamed_index"]} with index settings: {kwargs["index_settings"]} '

124 f'and ignoring settings: {kwargs["ignore_index_settings"]}. wait_for_completion: '

125 f'{kwargs["wait_for_completion"]}. Restore aliases: {aliases}. Current index '

126 f'name: {current_idx}'

127 )

128 self.loggit.info(msg)

129

130

131 def do_action(self):

132 """

133 Call :py:meth:`~.elasticsearch.client.SearchableSnapshotsClient.mount` to mount the indices

134 in :py:attr:`ilo` in the Frozen tier.

135

136 Verify index looks good

137

138 Call :py:meth:`~.elasticsearch.client.IndicesClient.update_aliases` to update each new

139 frozen index with the aliases from the old cold-tier index.

140

141 Verify aliases look good.

142

143 Call :py:meth:`~.elasticsearch.client.IndicesClient.delete` to delete the cold tier index.

144 """

145 try:

146 for kwargs in self.action_generator():

147 aliases = kwargs.pop('aliases')

148 current_idx = kwargs.pop('current_idx')

149 newidx = kwargs['renamed_index']

150 # Actually do the mount

151 self.loggit.debug('Mounting new index %s in frozen tier...', newidx)

152 self.client.searchable_snapshots.mount(**kwargs)

153 # Verify it's mounted as a partial now:

154 self.loggit.debug('Verifying new index %s is mounted properly...', newidx)

155 idx_settings = self.client.indices.get(index=newidx)[newidx]

156 if is_idx_partial(idx_settings['settings']['index']):

157 self.loggit.info('Index %s is mounted for frozen tier', newidx)

158 else:

159 raise SearchableSnapshotException(

160 f'Index {newidx} not a mounted searchable snapshot')

161 # Update Aliases

162 alias_names = aliases.keys()

163 if not alias_names:

164 self.loggit.warning('No aliases associated with index %s', current_idx)

165 else:

166 self.loggit.debug('Transferring aliases to new index %s', newidx)

167 self.client.indices.update_aliases(

168 actions=get_alias_actions(current_idx, newidx, aliases))

169 verify = self.client.indices.get(index=newidx)[newidx]['aliases'].keys()

170 if alias_names != verify:

171 self.loggit.error(

172 'Alias names do not match! %s does not match: %s', alias_names, verify)

173 raise FailedExecution('Aliases failed to transfer to new index')

174 # Clean up old index

175 self.loggit.debug('Deleting old index: %s', current_idx)

176 self.client.indices.delete(index=current_idx)

177 self.loggit.info(

178 'Successfully migrated %s to the frozen tier as %s', current_idx, newidx)

179

180 # pylint: disable=broad-except

181 except Exception as err:

182 report_failure(err)