Coverage for curator/helpers/testers.py: 100%

1"""Utility functions that get things"""

2 import logging

3 from voluptuous import Schema

4 from elasticsearch8 import Elasticsearch

5 from elasticsearch8.exceptions import NotFoundError

6 from es_client.helpers.utils import prune_nones

7 from curator.helpers.getters import get_repository, get_write_index

8 from curator.exceptions import (

9 ConfigurationError, MissingArgument, RepositoryException, SearchableSnapshotException)

10 from curator.defaults.settings import index_filtertypes, snapshot_actions, snapshot_filtertypes

11 from curator.validators import SchemaCheck, actions, options

12 from curator.validators.filter_functions import validfilters

13 from curator.helpers.utils import report_failure

14

15 def has_lifecycle_name(idx_settings):

16 """

17 :param idx_settings: The settings for an index being tested

18 :type idx_settings: dict

19

20 :returns: ``True`` if a lifecycle name exists in settings, else ``False``

21 :rtype: bool

22 """

23 if 'lifecycle' in idx_settings:

24 if 'name' in idx_settings['lifecycle']:

25 return True

26 return False

27

28 def is_idx_partial(idx_settings):

29 """

30 :param idx_settings: The settings for an index being tested

31 :type idx_settings: dict

32

33 :returns: ``True`` if store.snapshot.partial exists in settings, else ``False``

34 :rtype: bool

35 """

36 if 'store' in idx_settings:

37 if 'snapshot' in idx_settings['store']:

38 if 'partial' in idx_settings['store']['snapshot']:

39 if idx_settings['store']['snapshot']['partial']:

40 return True

41 # store.snapshot.partial exists but is False -- Not a frozen tier mount

42 return False

43 # store.snapshot exists, but partial isn't there -- Possibly a cold tier mount

44 return False

45 raise SearchableSnapshotException('Index not a mounted searchable snapshot')

46 raise SearchableSnapshotException('Index not a mounted searchable snapshot')

47

48 def ilm_policy_check(client, alias):

49 """Test if alias is associated with an ILM policy

50

51 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_settings`

52

53 :param client: A client connection object

54 :param alias: The alias name

55

56 :type client: :py:class:`~.elasticsearch.Elasticsearch`

57 :type alias: str

58 :rtype: bool

59 """

60 logger = logging.getLogger(__name__)

61 # alias = action_obj.options['name']

62 write_index = get_write_index(client, alias)

63 try:

64 idx_settings = client.indices.get_settings(index=write_index)

65 if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']:

66 # logger.info('Alias %s is associated with ILM policy.', alias)

67 # logger.info('Skipping action %s because allow_ilm_indices is false.', idx)

68 return True

69 except KeyError:

70 logger.debug('No ILM policies associated with %s', alias)

71 return False

72

73 def repository_exists(client, repository=None):

74 """

75 Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository`

76

77 :param client: A client connection object

78 :param repository: The Elasticsearch snapshot repository to use

79

80 :type client: :py:class:`~.elasticsearch.Elasticsearch`

81 :type repository: str

82

83 :returns: ``True`` if ``repository`` exists, else ``False``

84 :rtype: bool

85 """

86 logger = logging.getLogger(__name__)

87 if not repository:

88 raise MissingArgument('No value for "repository" provided')

89 try:

90 test_result = get_repository(client, repository)

91 if repository in test_result:

92 logger.debug("Repository %s exists.", repository)

93 response = True

94 else:

95 logger.debug("Repository %s not found...", repository)

96 response = False

97 # pylint: disable=broad-except

98 except Exception as err:

99 logger.debug('Unable to find repository "%s": Error: %s', repository, err)

100 response = False

101 return response

102

103 def rollable_alias(client, alias):

104 """

105 Calls :py:meth:`~.elasticsearch.client.IndicesClient.get_alias`

106

107 :param client: A client connection object

108 :param alias: An Elasticsearch alias

109

110 :type client: :py:class:`~.elasticsearch.Elasticsearch`

111 :type alias: str

112

113

114 :returns: ``True`` or ``False`` depending on whether ``alias`` is an alias that points to an

115 index that can be used by the ``_rollover`` API.

116 :rtype: bool

117 """

118 logger = logging.getLogger(__name__)

119 try:

120 response = client.indices.get_alias(name=alias)

121 except NotFoundError:

122 logger.error('Alias "%s" not found.', alias)

123 return False

124 # Response should be like:

125 # {'there_should_be_only_one': {'aliases': {'value of "alias" here': {}}}}

126 # where 'there_should_be_only_one' is a single index name that ends in a number, and 'value of

127 # "alias" here' reflects the value of the passed parameter, except where the ``is_write_index``

128 # setting makes it possible to have more than one index associated with a rollover index

129 for idx in response:

130 if 'is_write_index' in response[idx]['aliases'][alias]:

131 if response[idx]['aliases'][alias]['is_write_index']:

132 return True

133 # implied ``else``: If not ``is_write_index``, it has to fit the following criteria:

134 if len(response) > 1:

135 logger.error('"alias" must only reference one index, but points to %s', response)

136 return False

137 index = list(response.keys())[0]

138 rollable = False

139 # In order for `rollable` to be True, the last 2 digits of the index

140 # must be digits, or a hyphen followed by a digit.

141 # NOTE: This is not a guarantee that the rest of the index name is

142 # necessarily correctly formatted.

143 if index[-2:][1].isdigit():

144 if index[-2:][0].isdigit():

145 rollable = True

146 elif index[-2:][0] == '-':

147 rollable = True

148 return rollable

149

150 def snapshot_running(client):

151 """

152 Calls :py:meth:`~.elasticsearch.client.SnapshotClient.get_repository`

153

154 Return ``True`` if a snapshot is in progress, and ``False`` if not

155

156 :param client: A client connection object

157

158 :type client: :py:class:`~.elasticsearch.Elasticsearch`

159

160 :rtype: bool

161 """

162 try:

163 status = client.snapshot.status()['snapshots']

164 # pylint: disable=broad-except

165 except Exception as exc:

166 report_failure(exc)

167 # We will only accept a positively identified False. Anything else is

168 # suspect. That's why this statement, rather than just ``return status``

169 # pylint: disable=simplifiable-if-expression

170 return False if not status else True

171

172 def validate_actions(data):

173 """

174 Validate the ``actions`` configuration dictionary, as imported from actions.yml, for example.

175

176 :param data: The configuration dictionary

177

178 :type data: dict

179

180 :returns: The validated and sanitized configuration dictionary.

181 :rtype: dict

182 """

183 # data is the ENTIRE schema...

184 clean_config = {}

185 # Let's break it down into smaller chunks...

186 # First, let's make sure it has "actions" as a key, with a subdictionary

187 root = SchemaCheck(data, actions.root(), 'Actions File', 'root').result()

188 # We've passed the first step. Now let's iterate over the actions...

189 for action_id in root['actions']:

190 # Now, let's ensure that the basic action structure is correct, with

191 # the proper possibilities for 'action'

192 action_dict = root['actions'][action_id]

193 loc = f'Action ID "{action_id}"'

194 valid_structure = SchemaCheck(

195 action_dict, actions.structure(action_dict, loc), 'structure', loc

196 ).result()

197 # With the basic structure validated, now we extract the action name

198 current_action = valid_structure['action']

199 # And let's update the location with the action.

200 loc = f'Action ID "{action_id}", action "{current_action}"'

201 clean_options = SchemaCheck(

202 prune_nones(valid_structure['options']),

203 options.get_schema(current_action),

204 'options',

205 loc

206 ).result()

207 clean_config[action_id] = {

208 'action' : current_action,

209 'description' : valid_structure['description'],

210 'options' : clean_options,

211 }

212 if current_action == 'alias':

213 add_remove = {}

214 for k in ['add', 'remove']:

215 if k in valid_structure:

216 current_filters = SchemaCheck(

217 valid_structure[k]['filters'],

218 Schema(validfilters(current_action, location=loc)),

219 f'"{k}" filters',

220 f'{loc}, "filters"'

221 ).result()

222 add_remove.update(

223 {

224 k: {

225 'filters' : SchemaCheck(

226 current_filters,

227 Schema(validfilters(current_action, location=loc)),

228 'filters',

229 f'{loc}, "{k}", "filters"'

230 ).result()

231 }

232 }

233 )

234 # Add/Remove here

235 clean_config[action_id].update(add_remove)

236 elif current_action in ['cluster_routing', 'create_index', 'rollover']:

237 # neither cluster_routing nor create_index should have filters

238 pass

239 else: # Filters key only appears in non-alias actions

240 valid_filters = SchemaCheck(

241 valid_structure['filters'],

242 Schema(validfilters(current_action, location=loc)),

243 'filters',

244 f'{loc}, "filters"'

245 ).result()

246 clean_filters = validate_filters(current_action, valid_filters)

247 clean_config[action_id].update({'filters' : clean_filters})

248 # This is a special case for remote reindex

249 if current_action == 'reindex':

250 # Check only if populated with something.

251 if 'remote_filters' in valid_structure['options']:

252 valid_filters = SchemaCheck(

253 valid_structure['options']['remote_filters'],

254 Schema(validfilters(current_action, location=loc)),

255 'filters',

256 f'{loc}, "filters"'

257 ).result()

258 clean_remote_filters = validate_filters(current_action, valid_filters)

259 clean_config[action_id]['options'].update({'remote_filters': clean_remote_filters})

260

261 # if we've gotten this far without any Exceptions raised, it's valid!

262 return {'actions': clean_config}

263

264 def validate_filters(action, myfilters):

265 """

266 Validate that myfilters are appropriate for the action type, e.g. no

267 index filters applied to a snapshot list.

268

269 :param action: An action name

270 :param myfilters: A list of filters to test.

271

272 :type action: str

273 :type myfilters: list

274

275 :returns: Validated list of filters

276 :rtype: list

277 """

278 # Define which set of filtertypes to use for testing

279 if action in snapshot_actions():

280 filtertypes = snapshot_filtertypes()

281 else:

282 filtertypes = index_filtertypes()

283 for fil in myfilters:

284 if fil['filtertype'] not in filtertypes:

285 raise ConfigurationError(

286 f"\"{fil['filtertype']}\" filtertype is not compatible with action \"{action}\""

287 )

288 # If we get to this point, we're still valid. Return the original list

289 return myfilters

290

291 def verify_client_object(test):

292 """

293 :param test: The variable or object to test

294

295 :type test: :py:class:`~.elasticsearch.Elasticsearch`

296

297 :returns: ``True`` if ``test`` is a proper :py:class:`~.elasticsearch.Elasticsearch` client

298 object and raise a :py:exc:`TypeError` exception if it is not.

299 :rtype: bool

300 """

301 logger = logging.getLogger(__name__)

302 # Ignore mock type for testing

303 if str(type(test)) == "<class 'mock.Mock'>" or \

304 str(type(test)) == "<class 'mock.mock.Mock'>":

305 pass

306 elif not isinstance(test, Elasticsearch):

307 msg = f'Not a valid client object. Type: {type(test)} was passed'

308 logger.error(msg)

309 raise TypeError(msg)

310

311 def verify_index_list(test):

312 """

313 :param test: The variable or object to test

314

315 :type test: :py:class:`~.curator.IndexList`

316

317 :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.indexlist.IndexList` object,

318 else raise a :py:class:`TypeError` exception.

319 :rtype: None

320 """

321 # It breaks if this import isn't local to this function:

322 # ImportError: cannot import name 'IndexList' from partially initialized module

323 # 'curator.indexlist' (most likely due to a circular import)

324 # pylint: disable=import-outside-toplevel

325 from curator.indexlist import IndexList

326 logger = logging.getLogger(__name__)

327 if not isinstance(test, IndexList):

328 msg = f'Not a valid IndexList object. Type: {type(test)} was passed'

329 logger.error(msg)

330 raise TypeError(msg)

331

332 def verify_repository(client, repository=None):

333 """

334 Do :py:meth:`~.elasticsearch.snapshot.verify_repository` call. If it fails, raise a

335 :py:exc:`~.curator.exceptions.RepositoryException`.

336

337 :param client: A client connection object

338 :type client: :py:class:`~.elasticsearch.Elasticsearch`

339 :param repository: A repository name

340

341 :type client: :py:class:`~.elasticsearch.Elasticsearch`

342 :type repository: str

343

344 :rtype: None

345 """

346 logger = logging.getLogger(__name__)

347 try:

348 nodes = client.snapshot.verify_repository(name=repository)['nodes']

349 logger.debug('All nodes can write to the repository')

350 logger.debug('Nodes with verified repository access: %s', nodes)

351 except Exception as err:

352 try:

353 if err.status_code == 404:

354 msg = (

355 f'--- Repository "{repository}" not found. Error: '

356 f'{err.meta.status}, {err.error}'

357 )

358 else:

359 msg = (

360 f'--- Got a {err.meta.status} response from Elasticsearch. '

361 f'Error message: {err.error}'

362 )

363 except AttributeError:

364 msg = f'--- Error message: {err}'.format()

365 report = f'Failed to verify all nodes have repository access: {msg}'

366 raise RepositoryException(report) from err

367

368 def verify_snapshot_list(test):

369 """

370 :param test: The variable or object to test

371

372 :type test: :py:class:`~.curator.SnapshotList`

373

374 :returns: ``None`` if ``test`` is a proper :py:class:`~.curator.snapshotlist.SnapshotList`

375 object, else raise a :py:class:`TypeError` exception.

376 :rtype: None

377 """

378 # It breaks if this import isn't local to this function:

379 # ImportError: cannot import name 'SnapshotList' from partially initialized module

380 # 'curator.snapshotlist' (most likely due to a circular import)

381 # pylint: disable=import-outside-toplevel

382 from curator.snapshotlist import SnapshotList

383 logger = logging.getLogger(__name__)

384 if not isinstance(test, SnapshotList):

385 msg = f'Not a valid SnapshotList object. Type: {type(test)} was passed'

386 logger.error(msg)

387 raise TypeError(msg)