Coverage for /Users/buh/.pyenv/versions/3.12.2/envs/pii/lib/python3.12/site-packages/es_pii_tool/helpers/steps.py: 71%

1"""Each function is a single step in PII redaction"""

2

3 import typing as t

4 import time

5 import logging

6 from dotmap import DotMap # type: ignore

7 from es_wait import IlmPhase, IlmStep

8 from es_pii_tool.defaults import PAUSE_DEFAULT

9 from es_pii_tool.exceptions import (

10 BadClientResult,

11 FatalError,

12 MissingArgument,

13 MissingError,

14 MissingIndex,

15 ValueMismatch,

16)

17 from es_pii_tool.trackables import Step

18 from es_pii_tool.helpers import elastic_api as api

19 from es_pii_tool.helpers.utils import (

20 configure_ilm_policy,

21 get_alias_actions,

22 strip_ilm_name,

23 es_waiter,

24 timing,

25)

26

27 if t.TYPE_CHECKING:

28 from es_pii_tool.trackables import Task

29

30 logger = logging.getLogger(__name__)

31

32

33 def failed_step(task: 'Task', step: 'Step', exc):

34 """Function to avoid repetition of code if a step fails"""

35 # MissingIndex, BadClientResult are the only ones inbound

36 if isinstance(exc, MissingIndex):

37 msg = (

38 f'Step failed because index {exc.missing} was not found. The upstream '

39 f'exception type was MissingIndex, with error message: '

40 f'{exc.upstream.args[0]}'

41 )

42 elif isinstance(exc, BadClientResult):

43 msg = (

44 f'Step failed because of a bad or unexpected response or result from '

45 f'the Elasticsearch cluster. The upstream exception type was '

46 f'BadClientResult, with error message: {exc.upstream.args[0]}'

47 )

48 else:

49 msg = f'Step failed for an unexpected reason: {exc}'

50 logger.critical(msg)

51 step.end(False, errors=True, logmsg=f'{msg}')

52 task.end(False, errors=True, logmsg=f'Failed {step.stepname}')

53 raise FatalError(msg, exc)

54

55

56 def metastep(task: 'Task', stepname: str, func, *args, **kwargs) -> None:

57 """The reusable step"""

58 step = Step(task=task, stepname=stepname)

59 if step.finished():

60 logger.info('%s: already completed', step.stub)

61 return

62 step.begin()

63 dry_run_safe = kwargs.pop('dry_run_safe', False)

64 dry_run_msg = kwargs.pop('dry_run_msg', None)

65 include_step = kwargs.pop('include_step', False)

66 if include_step:

67 kwargs['step'] = step

68 if (dry_run_safe and task.job.dry_run) or not task.job.dry_run:

69 try:

70 response = func(*args, **kwargs)

71 except (MissingIndex, BadClientResult, ValueMismatch) as exc:

72 failed_step(task, step, exc)

73 if response:

74 step.add_log(f'{response}')

75 else:

76 if dry_run_msg is None:

77 dry_run_msg = 'No action logged'

78 msg = f'Dry-Run: No changes, but expected behavior: {dry_run_msg}'

79 step.add_log(msg)

80 logger.debug(msg)

81 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

82

83

84 def missing_data(stepname, kwargs) -> None:

85 """Avoid duplicated code for data check"""

86 if 'data' not in kwargs:

87 msg = f'"{stepname}" is missing keyword argument(s)'

88 what = 'type: DotMap'

89 names = ['data']

90 raise MissingArgument(msg, what, names)

91

92

93 def _meta_resolve_index(var: DotMap, data: DotMap) -> str:

94 """Make a metastep for resolve_index"""

95 result = api.resolve_index(var.client, var.index)

96 logger.debug('resolve data: %s', result)

97 response = ''

98 try:

99 data.data_stream = result['indices'][0]['data_stream']

100 except KeyError:

101 response = f'Index {var.index} is not part of a data_stream'

102 logger.debug(response)

103 return response

104

105

106 def resolve_index(task: 'Task', stepname: str, var: DotMap, **kwargs) -> None:

107 """

108 Resolve the index to see if it's part of a data stream

109 """

110 missing_data(stepname, kwargs)

111 data = kwargs['data']

112 metastep(task, stepname, _meta_resolve_index, var, data, dry_run_safe=True)

113

114

115 def _meta_pre_delete(var: DotMap) -> str:

116 """Make a metastep for pre_delete"""

117 response = ''

118 # The metastep will handle the "don't do this if dry_run" logic

119 try:

120 api.delete_index(var.client, var.redaction_target)

121 except MissingIndex:

122 # Not a problem. This is normal and expected.

123 response = f'Pre-delete did not find index "{var.redaction_target}"'

124 logger.debug(response)

125 return response

126

127

128 def pre_delete(task: 'Task', stepname: str, var: DotMap, **kwargs) -> None:

129 """

130 Pre-delete the redacted index to ensure no collisions. Ignore if not present

131 """

132 missing_data(stepname, kwargs)

133 drm = 'Delete index {var.redaction_target} (if it exists)'

134 metastep(task, stepname, _meta_pre_delete, var, dry_run_msg=drm)

135

136

137 def _meta_restore_index(var: DotMap) -> str:

138 """Make a metastep for restore_index"""

139 response = f'Restored {var.ss_idx} to {var.redaction_target}'

140 try:

141 api.restore_index(

142 var.client,

143 var.repository,

144 var.ss_snap,

145 var.ss_idx,

146 var.redaction_target,

147 index_settings=var.restore_settings.toDict(),

148 )

149 except BadClientResult as bad:

150 response = f'Unable to restore {var.ss_idx} to {var.redaction_target}: {bad}'

151 logger.error(response)

152 return response

153

154

155 def restore_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

156 """Restore index from snapshot"""

157 missing_data(stepname, kwargs)

158 drm = f'Restore {var.ss_idx} to {var.redaction_target}'

159 metastep(task, stepname, _meta_restore_index, var, dry_run_msg=drm)

160

161

162 def _meta_get_ilm_data(var: DotMap, data: DotMap) -> str:

163 """Make a metastep for get_index_lifecycle_data"""

164 res = api.get_settings(var.client, var.index)

165 response = ''

166 data.index = DotMap()

167 data.index.lifecycle = DotMap(

168 {'name': None, 'rollover_alias': None, 'indexing_complete': True}

169 )

170 try:

171 data.index.lifecycle = DotMap(res[var.index]['settings']['index']['lifecycle'])

172 except KeyError as err:

173 response = f'Index {var.index} missing one or more lifecycle keys: {err}'

174 if data.index.lifecycle.name:

175 response = f'Index lifecycle settings: {data.index.lifecycle}'

176 else:

177 response = f'Index {var.index} has no ILM lifecycle'

178 logger.debug(response)

179 return response

180

181

182 def get_index_lifecycle_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

183 """

184 Populate data.index with index settings results referenced at

185 INDEXNAME.settings.index.lifecycle

186 """

187 missing_data(stepname, kwargs)

188 data = kwargs['data']

189 metastep(task, stepname, _meta_get_ilm_data, var, data, dry_run_safe=True)

190

191

192 def _meta_get_ilm_explain_data(var: DotMap, data: DotMap) -> str:

193 """Make a metastep for get_ilm_explain_data"""

194 response = ''

195 if data.index.lifecycle.name:

196 data.ilm = DotMap()

197 try:

198 res = api.get_ilm(var.client, var.index)

199 data.ilm.explain = DotMap(res['indices'][var.index])

200 response = f'ILM explain settings: {data.ilm.explain}'

201 except MissingIndex as exc:

202 logger.error('Index %s not found in ILM explain data', var.index)

203 raise exc

204 else:

205 response = f'Index {var.index} has no ILM explain data'

206 logger.debug(response)

207 return response

208

209

210 def get_ilm_explain_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

211 """

212 Populate data.ilm.explain with ilm_explain data

213 """

214 missing_data(stepname, kwargs)

215 data = kwargs['data']

216 metastep(task, stepname, _meta_get_ilm_explain_data, var, data, dry_run_safe=True)

217

218

219 def _meta_get_ilm_lifecycle_data(var: DotMap, data: DotMap) -> str:

220 """Make a metastep for get_ilm_lifecycle_data"""

221 response = ''

222 if data.index.lifecycle.name:

223 res = api.get_ilm_lifecycle(var.client, data.index.lifecycle.name)

224 if not res:

225 msg = f'No such ILM policy: {data.index.lifecycle.name}'

226 raise BadClientResult(msg, Exception())

227 data.ilm.lifecycle = DotMap(res[data.index.lifecycle.name])

228 response = f'ILM lifecycle settings: {data.ilm.lifecycle}'

229 else:

230 response = f'Index {var.index} has no ILM lifecycle data'

231 logger.debug(response)

232 return response

233

234

235 def get_ilm_lifecycle_data(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

236 """

237 Populate data.ilm.explain with ilm_explain data

238 """

239 missing_data(stepname, kwargs)

240 data = kwargs['data']

241 metastep(task, stepname, _meta_get_ilm_lifecycle_data, var, data, dry_run_safe=True)

242

243

244 def clone_ilm_policy(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

245 """

246 If this index has an ILM policy, we need to clone it so we can attach

247 the new index to it.

248 """

249 missing_data(stepname, kwargs)

250 data = kwargs['data']

251 step = Step(task=task, stepname=stepname)

252 if step.finished():

253 logger.info('%s: already completed', step.stub)

254 return

255 step.begin()

256 if data.index.lifecycle.name is None or not data.ilm.lifecycle.policy:

257 _ = f'{stepname}: Index {var.index} has no ILM lifecycle or policy data'

258 logger.debug(_)

259 step.add_log(_)

260 return

261 data.new = DotMap()

262

263 # From here, we check for matching named cloned policy

264

265 configure_ilm_policy(task, data)

266

267 # New ILM policy naming: pii-tool-POLICYNAME---v###

268 stub = f'pii-tool-{strip_ilm_name(data.index.lifecycle.name)}'

269 policy = data.new.ilmpolicy.toDict() # For comparison

270 resp = {'dummy': 'startval'} # So the while loop can start with something

271 policyver = 0 # Our version number starting point.

272 policymatch = False

273 while resp:

274 data.new.ilmname = f'{stub}---v{policyver + 1:03}'

275 resp = api.get_ilm_lifecycle(var.client, data.new.ilmname) # type: ignore

276 if resp: # We have data, so the name matches

277 # Compare the new policy to the one just returned

278 if policy == resp[data.new.ilmname]['policy']: # type: ignore

279 msg = f'New policy data matches: {data.new.ilmname}'

280 logger.debug(msg)

281 step.add_log(msg)

282 policymatch = True

283 break # We can drop out of the loop here.

284 # Implied else: resp has no value, so the while loop will end.

285 policyver += 1

286 msg = f'New ILM policy name (may already exist): {data.new.ilmname}'

287 logger.debug(msg)

288 step.add_log(msg)

289 if not task.job.dry_run: # Don't create if dry_run

290 if not policymatch:

291 # Create the cloned ILM policy

292 try:

293 gkw = {'name': data.new.ilmname, 'policy': policy}

294 api.generic_get(var.client.ilm.put_lifecycle, **gkw)

295 except (MissingError, BadClientResult) as exc:

296 _ = f'Unable to put new ILM policy: {exc}'

297 logger.error(_)

298 step.add_log(_)

299 failed_step(task, step, exc)

300 # Implied else: We've arrived at the expected new ILM name

301 # and it does match an existing policy in name and content

302 # so we don't need to create a new one.

303 else:

304 _ = (

305 f'Dry-Run: No changes, but expected behavior: '

306 f'ILM policy {data.new.ilmname} created'

307 )

308 logger.debug(_)

309 step.add_log(_)

310 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

311

312

313 def un_ilm_the_restored_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

314 """Remove the lifecycle data from the settings of the restored index"""

315 missing_data(stepname, kwargs)

316 drm = f'Any existing ILM policy removed from {var.redaction_target}'

317 metastep(

318 task,

319 stepname,

320 api.remove_ilm_policy,

321 var.client,

322 var.redaction_target,

323 dry_run_msg=drm,

324 )

325

326

327 def redact_from_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

328 """Run update by query on new restored index"""

329 missing_data(stepname, kwargs)

330 drm = (

331 f'Redact index {var.redaction_target} replacing content of fields: '

332 f'{task.job.config["fields"]} with message: {task.job.config["message"]}'

333 )

334 metastep(

335 task,

336 stepname,

337 api.redact_from_index,

338 var.client,

339 var.redaction_target,

340 task.job.config,

341 dry_run_msg=drm,

342 )

343

344

345 def _meta_forcemerge_index(task: 'Task', var: DotMap, **kwargs) -> str:

346 """Do some task logging around the forcemerge api call"""

347 step = kwargs.pop('step', None)

348 if step is None:

349 raise MissingArgument('_meta_forcemerge_index', 'keyword argument', 'step')

350 index = var.redaction_target

351 msg = f'Before forcemerge, {api.report_segment_count(var.client, index)}'

352 logger.info(msg)

353 step.add_log(msg)

354 fmkwargs = {}

355 if 'forcemerge' in task.job.config:

356 fmkwargs = task.job.config['forcemerge']

357 fmkwargs['index'] = index

358 if 'only_expunge_deletes' in fmkwargs and fmkwargs['only_expunge_deletes']:

359 msg = 'Forcemerge will only expunge deleted docs!'

360 logger.info(msg)

361 step.add_log(msg)

362 else:

363 mns = 1 # default value

364 if 'max_num_segments' in fmkwargs and isinstance(

365 fmkwargs['max_num_segments'], int

366 ):

367 mns = fmkwargs['max_num_segments']

368 msg = f'Proceeding to forcemerge to {mns} segments per shard'

369 logger.info(msg)

370 step.add_log(msg)

371 logger.debug('forcemerge kwargs = %s', fmkwargs)

372 # Do the actual forcemerging

373 api.forcemerge_index(var.client, **fmkwargs)

374 msg = f'After forcemerge, {api.report_segment_count(var.client, index)}'

375 logger.info(msg)

376 step.add_log(msg)

377 return msg

378

379

380 def forcemerge_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

381 """Force merge redacted index"""

382 missing_data(stepname, kwargs)

383 msg = ''

384 fmkwargs = {}

385 if 'forcemerge' in task.job.config:

386 fmkwargs = task.job.config['forcemerge']

387 if 'only_expunge_deletes' in fmkwargs and fmkwargs['only_expunge_deletes']:

388 msg = 'only expunging deleted docs'

389 else:

390 mns = 1 # default value

391 if 'max_num_segments' in fmkwargs and isinstance(

392 fmkwargs['max_num_segments'], int

393 ):

394 mns = fmkwargs['max_num_segments']

395 msg = f'to {mns} segments per shard'

396 drm = f'Forcemerge index {var.redaction_target} {msg}'

397 metastep(

398 task,

399 stepname,

400 _meta_forcemerge_index,

401 task,

402 var,

403 include_step=True,

404 dry_run_msg=drm,

405 )

406

407

408 def clear_cache(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

409 """Clear cache of redacted index"""

410 missing_data(stepname, kwargs)

411 drm = f'Clear cache of index {var.redaction_target}'

412 metastep(

413 task,

414 stepname,

415 api.clear_cache,

416 var.client,

417 var.redaction_target,

418 dry_run_msg=drm,

419 )

420

421

422 def confirm_redaction(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

423 """Check update by query did its job"""

424 missing_data(stepname, kwargs)

425 drm = f'Confirm redaction of index {var.redaction_target}'

426 metastep(

427 task,

428 stepname,

429 api.check_index,

430 var.client,

431 var.redaction_target,

432 task.job.config,

433 dry_run_msg=drm,

434 )

435

436

437 def snapshot_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

438 """Create a new snapshot for mounting our redacted index"""

439 missing_data(stepname, kwargs)

440 drm = f'Snapshot index {var.redaction_target} to {var.new_snap_name}'

441 metastep(

442 task,

443 stepname,

444 api.take_snapshot,

445 var.client,

446 var.repository,

447 var.new_snap_name,

448 var.redaction_target,

449 dry_run_msg=drm,

450 )

451

452

453 def mount_snapshot(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

454 """

455 Mount the index as a searchable snapshot to make the redacted index available

456 """

457 missing_data(stepname, kwargs)

458 drm = (

459 f'Mount index {var.redaction_target} in snapshot '

460 f'{var.new_snap_name} as {var.mount_name}'

461 )

462 metastep(task, stepname, api.mount_index, var, dry_run_msg=drm)

463

464

465 def apply_ilm_policy(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

466 """

467 If the index was associated with an ILM policy, associate it with the

468 new, cloned ILM policy.

469 """

470 missing_data(stepname, kwargs)

471 data = kwargs['data']

472 if data.new.ilmname:

473 settings = {'index': {}} # type: ignore

474 # Add all of the original lifecycle settings

475 settings['index']['lifecycle'] = data.index.lifecycle.toDict()

476 # Replace the name with the new ILM policy name

477 settings['index']['lifecycle']['name'] = data.new.ilmname

478 drm = f'Apply new ILM policy {data.new.ilmname} to {var.mount_name}'

479 metastep(

480 task,

481 stepname,

482 api.put_settings,

483 var.client,

484 var.mount_name,

485 settings,

486 dry_run_msg=drm,

487 )

488

489

490 def confirm_ilm_phase(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

491 """

492 Confirm the mounted index is in the expected ILM phase

493 This is done by using move_to_step. If it's already in the step, no problem.

494 If it's in step ``new``, this will advance the index to the expected step.

495 """

496 missing_data(stepname, kwargs)

497 step = Step(task=task, stepname=stepname)

498 if step.finished():

499 logger.info('%s: already completed', step.stub)

500 return

501 step.begin()

502 if task.job.dry_run:

503 msg = f'Dry-Run: {var.mount_name} moved to ILM phase {var.phase}'

504 logger.debug(msg)

505 step.add_log(msg)

506 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

507 return

508 # Wait for phase to be "new"

509 pause, timeout = timing('ilm')

510 logger.debug(f'ENV pause = {pause}, timeout = {timeout}')

511 try:

512 # Update in es_wait 0.9.2:

513 # - If you send phase='new', it will wait for the phase to be 'new' or higher

514 # - This is where a user was getting stuck. They were waiting for 'new' but

515 # - the phase was already 'frozen', so it was endlessly checking for 'new'.

516 es_waiter(

517 var.client,

518 IlmPhase,

519 name=var.mount_name,

520 phase='new',

521 pause=pause,

522 timeout=timeout,

523 )

524 # Wait for step to be "complete"

525 es_waiter(

526 var.client, IlmStep, name=var.mount_name, pause=pause, timeout=timeout

527 )

528 except BadClientResult as bad:

529 _ = f'ILM step confirmation problem -- ERROR: {bad}'

530 logger.error(_)

531 step.add_log(_)

532 failed_step(task, step, bad)

533

534 def get_currstep():

535 try:

536 _ = api.generic_get(var.client.ilm.explain_lifecycle, index=var.mount_name)

537 except MissingError as exc:

538 _ = f'Unable to get ILM phase of {var.mount_name}'

539 logger.error(_)

540 step.add_log(_)

541 failed_step(task, step, exc)

542 try:

543 expl = _['indices'][var.mount_name]

544 except KeyError as err:

545 msg = f'{var.mount_name} not found in ILM explain data: {err}'

546 logger.error(msg)

547 step.add_log(msg)

548 failed_step(task, step, err)

549 if 'managed' not in expl:

550 msg = f'Index {var.mount_name} is not managed by ILM'

551 step.add_log(msg)

552 failed_step(

553 task, step, ValueMismatch(msg, expl['managed'], '{"managed": True}')

554 )

555 return {'phase': expl['phase'], 'action': expl['action'], 'name': expl['step']}

556

557 nextstep = {'phase': var.phase, 'action': 'complete', 'name': 'complete'}

558 if task.job.dry_run: # Don't actually move_to_step if dry_run

559 msg = (

560 f'{stepname}: Dry-Run: {var.mount_name} not moved/confirmed to ILM '

561 f'phase {var.phase}'

562 )

563 logger.debug(msg)

564 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

565 return

566

567 # We will try to move the index to the expected phase up to 3 times

568 # before failing the step.

569 attempts = 0

570 success = False

571 while attempts < 3 and not success:

572 # Since we are now testing for 'new' or higher, we may not need to advance

573 # ILM phases. If the current step is already where we expect to be, log

574 # confirmation and move on.

575 logger.debug('Attempt number: %s', attempts)

576 currstep = get_currstep()

577 if currstep == nextstep:

578 msg = (

579 f'{stepname}: {var.mount_name} is confirmed to be in ILM phase '

580 f'{var.phase}'

581 )

582 logger.debug(msg)

583 step.add_log(msg)

584 # Set both while loop critera to values that will end the loop

585 success = True

586 attempts = 3

587 else:

588 # If we are not yet in the expected target phase, then proceed with the

589 # ILM phase change.

590 logger.debug('Current ILM Phase: %s', currstep)

591 logger.debug('Target ILM Phase: %s', nextstep)

592 logger.debug('PHASE: %s', var.phase)

593 try:

594 api.ilm_move(var.client, var.mount_name, currstep, nextstep)

595 success = True

596 except BadClientResult as bad:

597 logger.debug('Attempt failed. Incrementing attempts.')

598 attempts += 1

599 if attempts == 3:

600 _ = 'Attempt limit reached. Failing step.'

601 logger.error(_)

602 step.add_log(_)

603 failed_step(task, step, bad)

604 logger.debug('Waiting %s seconds before retrying...', PAUSE_DEFAULT)

605 time.sleep(float(PAUSE_DEFAULT))

606 logger.warning('ILM move failed: %s -- Retrying...', bad.message)

607 continue

608 pause, timeout = timing('ilm')

609 logger.debug(f'ENV pause = {pause}, timeout = {timeout}')

610 try:

611 es_waiter(

612 var.client,

613 IlmPhase,

614 name=var.mount_name,

615 phase=var.phase,

616 pause=pause,

617 timeout=timeout,

618 )

619 es_waiter(

620 var.client,

621 IlmStep,

622 name=var.mount_name,

623 pause=pause,

624 timeout=timeout,

625 )

626 except BadClientResult as phase_err:

627 msg = f'Unable to wait for ILM step to complete -- ERROR: {phase_err}'

628 logger.error(msg)

629 step.add_log(msg)

630 failed_step(task, step, phase_err)

631 # If we make it here, we have successfully moved the index to the expected phase

632 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

633

634

635 def delete_redaction_target(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

636 """

637 Now that it's mounted (with a new name), we should delete the redaction_target

638 index

639 """

640 missing_data(stepname, kwargs)

641 drm = f'Delete redaction target index {var.redaction_target}'

642 metastep(

643 task,

644 stepname,

645 api.delete_index,

646 var.client,

647 var.redaction_target,

648 dry_run_msg=drm,

649 )

650

651

652 def fix_aliases(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

653 """Using the aliases collected from var.index, update mount_name and verify"""

654 missing_data(stepname, kwargs)

655 data = kwargs['data']

656 step = Step(task=task, stepname=stepname)

657 if step.finished():

658 logger.info('%s: already completed', step.stub)

659 return

660 step.begin()

661 if data.data_stream:

662 msg = 'Cannot apply aliases to indices in data_stream'

663 logger.debug(msg)

664 step.add_log(msg)

665 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

666 return

667 alias_names = var.aliases.toDict().keys()

668 if not alias_names:

669 msg = f'No aliases associated with index {var.index}'

670 step.add_log(msg)

671 logger.info(msg)

672 elif not task.job.dry_run:

673 msg = f'Transferring aliases to new index ' f'{var.mount_name}'

674 logger.debug(msg)

675 step.add_log(msg)

676 var.client.indices.update_aliases(

677 actions=get_alias_actions(var.index, var.mount_name, var.aliases.toDict())

678 )

679 verify = var.client.indices.get(index=var.mount_name)[var.mount_name][

680 'aliases'

681 ].keys()

682 if alias_names != verify:

683 msg = f'Alias names do not match! {alias_names} does not match: {verify}'

684 logger.critical(msg)

685 step.add_log(msg)

686 failed_step(

687 task, step, ValueMismatch(msg, 'alias names mismatch', alias_names)

688 )

689 else:

690 msg = 'Dry-Run: alias transfer not executed'

691 logger.debug(msg)

692 step.add_log(msg)

693 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

694

695

696 def un_ilm_the_original_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

697 """

698 Remove the lifecycle data from the settings of the original index

699

700 This is chiefly done as a safety measure.

701 """

702 missing_data(stepname, kwargs)

703 metastep(task, stepname, api.remove_ilm_policy, var.client, var.index)

704

705

706 def close_old_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

707 """Close old mounted snapshot"""

708 missing_data(stepname, kwargs)

709 metastep(task, stepname, api.close_index, var.client, var.index)

710

711

712 def delete_old_index(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

713 """Delete old mounted snapshot, if configured to do so"""

714 missing_data(stepname, kwargs)

715 step = Step(task=task, stepname=stepname)

716 if step.finished():

717 logger.info('%s: already completed', step.stub)

718 return

719 step.begin()

720 if task.job.config['delete']:

721 msg = f'Deleting original mounted index: {var.index}'

722 task.add_log(msg)

723 logger.info(msg)

724 try:

725 api.delete_index(var.client, var.index)

726 except MissingIndex as miss:

727 msg = f'Index {var.index} not found for deletion: {miss}'

728 logger.error(msg)

729 step.add_log(msg)

730 except BadClientResult as bad:

731 msg = f'Bad client result: {bad}'

732 logger.error(msg)

733 step.add_log(msg)

734 failed_step(task, step, bad)

735 else:

736 msg = (

737 f'delete set to False — not deleting original mounted index: '

738 f'{var.index}'

739 )

740 task.add_log(msg)

741 logger.warning(msg)

742 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

743

744

745 def assign_aliases(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

746 """Put the starting index name on new mounted index as alias"""

747 missing_data(stepname, kwargs)

748 data = kwargs['data']

749 step = Step(task=task, stepname=stepname)

750 if step.finished():

751 logger.info('%s: already completed', step.stub)

752 return

753 step.begin()

754 if data.data_stream:

755 msg = 'Cannot apply aliases to indices in data_stream'

756 logger.debug(msg)

757 step.add_log(msg)

758 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

759 return

760 if not task.job.dry_run:

761 msg = f'Assigning aliases {var.index} to index {var.mount_name}'

762 logger.debug(msg)

763 step.add_log(msg)

764 try:

765 api.assign_alias(var.client, var.mount_name, var.index)

766 except BadClientResult as bad:

767 failed_step(task, step, bad)

768 else:

769 msg = f'Assigning aliases {var.index} to index {var.mount_name}'

770 _ = f'Dry-Run: No changes, but expected behavior: {msg}'

771 logger.debug(_)

772 step.add_log(_)

773 step.end(completed=True, errors=False, logmsg=f'{stepname} completed')

774

775

776 def reassociate_index_with_ds(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

777 """

778 If the index was associated with a data_stream, reassociate it with the

779 data_stream again.

780 """

781 missing_data(stepname, kwargs)

782 data = kwargs['data']

783 acts = [{'add_backing_index': {'index': var.mount_name}}]

784 if data.data_stream:

785 acts[0]['add_backing_index']['data_stream'] = data.data_stream

786 logger.debug('%s: Modify data_stream actions: %s', stepname, acts)

787 drm = f'Reassociate index {var.mount_name} with data_stream {data.data_stream}'

788 metastep(

789 task, stepname, api.modify_data_stream, var.client, acts, dry_run_msg=drm

790 )

791

792

793 def _meta_record_it(task: 'Task', snapname: str) -> str:

794 """Make a metastep for record_it"""

795 task.job.cleanup.append(snapname)

796 return f'Snapshot {snapname} added to cleanup list'

797

798

799 def record_it(task: 'Task', stepname, var: DotMap, **kwargs) -> None:

800 """Record the now-deletable snapshot in the job's tracking index."""

801 missing_data(stepname, kwargs)

802 drm = f'Snapshot {var.ss_snap} added to cleanup list'

803 metastep(task, stepname, _meta_record_it, task, var.ss_snap, dry_run_msg=drm)