Coverage for curator/cli.py: 99%

1"""Main CLI for Curator"""

2 import os

3 import sys

4 import logging

5 import yaml

6 import click

7 from voluptuous import Schema

8 from curator import actions

9 from curator.config_utils import process_config, password_filter

10 from curator.defaults import settings

11 from curator.exceptions import ClientException, ConfigurationError, NoIndices, NoSnapshots

12 from curator.indexlist import IndexList

13 from curator.snapshotlist import SnapshotList

14 from curator.utils import get_client, get_yaml, prune_nones, validate_actions, get_write_index

15 from curator.validators import SchemaCheck

16 from curator._version import __version__

17

18 CLASS_MAP = {

19 'alias' : actions.Alias,

20 'allocation' : actions.Allocation,

21 'close' : actions.Close,

22 'cluster_routing' : actions.ClusterRouting,

23 'create_index' : actions.CreateIndex,

24 'delete_indices' : actions.DeleteIndices,

25 'delete_snapshots' : actions.DeleteSnapshots,

26 'forcemerge' : actions.ForceMerge,

27 'index_settings' : actions.IndexSettings,

28 'open' : actions.Open,

29 'reindex' : actions.Reindex,

30 'replicas' : actions.Replicas,

31 'restore' : actions.Restore,

32 'rollover' : actions.Rollover,

33 'snapshot' : actions.Snapshot,

34 'shrink' : actions.Shrink,

35}

36

37 def process_action(client, config, **kwargs):

38 """

39 Do the `action` in the configuration dictionary, using the associated args.

40 Other necessary args may be passed as keyword arguments

41

42 :arg config: An `action` dictionary.

43 """

44 logger = logging.getLogger(__name__)

45 # Make some placeholder variables here for readability

46 logger.debug('Configuration dictionary: {0}'.format(config))

47 logger.debug('kwargs: {0}'.format(kwargs))

48 action = config['action']

49 # This will always have some defaults now, so no need to do the if...

50 # # OLD WAY: opts = config['options'] if 'options' in config else {}

51 opts = config['options']

52 logger.debug('opts: {0}'.format(opts))

53 mykwargs = {}

54

55 action_class = CLASS_MAP[action]

56

57 # Add some settings to mykwargs...

58 if action == 'delete_indices':

59 mykwargs['master_timeout'] = (

60 kwargs['master_timeout'] if 'master_timeout' in kwargs else 30)

61

62 ### Update the defaults with whatever came with opts, minus any Nones

63 mykwargs.update(prune_nones(opts))

64 logger.debug('Action kwargs: {0}'.format(mykwargs))

65

66 ### Set up the action ###

67 if action == 'alias':

68 # Special behavior for this action, as it has 2 index lists

69 logger.debug('Running "{0}" action'.format(action.upper()))

70 action_obj = action_class(**mykwargs)

71 removes = IndexList(client)

72 adds = IndexList(client)

73 if 'remove' in config:

74 logger.debug(

75 'Removing indices from alias "{0}"'.format(opts['name']))

76 removes.iterate_filters(config['remove'])

77 action_obj.remove(

78 removes, warn_if_no_indices=opts['warn_if_no_indices'])

79 if 'add' in config:

80 logger.debug('Adding indices to alias "{0}"'.format(opts['name']))

81 adds.iterate_filters(config['add'])

82 action_obj.add(adds, warn_if_no_indices=opts['warn_if_no_indices'])

83 elif action in ['cluster_routing', 'create_index', 'rollover']:

84 action_obj = action_class(client, **mykwargs)

85 elif action == 'delete_snapshots' or action == 'restore':

86 logger.debug('Running "{0}"'.format(action))

87 slo = SnapshotList(client, repository=opts['repository'])

88 slo.iterate_filters(config)

89 # We don't need to send this value to the action

90 mykwargs.pop('repository')

91 action_obj = action_class(slo, **mykwargs)

92 else:

93 logger.debug('Running "{0}"'.format(action.upper()))

94 ilo = IndexList(client)

95 ilo.iterate_filters(config)

96 action_obj = action_class(ilo, **mykwargs)

97 ### Do the action

98 if 'dry_run' in kwargs and kwargs['dry_run']:

99 action_obj.do_dry_run()

100 else:

101 logger.debug('Doing the action here.')

102 action_obj.do_action()

103

104 def run(config, action_file, dry_run=False):

105 """

106 Actually run.

107 """

108 client_args = process_config(config)

109 logger = logging.getLogger(__name__)

110 logger.debug('Client and logging options validated.')

111

112 # Extract this and save it for later, in case there's no timeout_override.

113 default_timeout = client_args.pop('timeout')

114 logger.debug('default_timeout = {0}'.format(default_timeout))

115 #########################################

116 ### Start working on the actions here ###

117 #########################################

118 logger.debug('action_file: {0}'.format(action_file))

119 action_config = get_yaml(action_file)

120 logger.debug('action_config: {0}'.format(password_filter(action_config)))

121 action_dict = validate_actions(action_config)

122 actions = action_dict['actions']

123 logger.debug('Full list of actions: {0}'.format(password_filter(actions)))

124 action_keys = sorted(list(actions.keys()))

125 for idx in action_keys:

126 action = actions[idx]['action']

127 action_disabled = actions[idx]['options'].pop('disable_action')

128 logger.debug('action_disabled = {0}'.format(action_disabled))

129 continue_if_exception = (

130 actions[idx]['options'].pop('continue_if_exception'))

131 logger.debug(

132 'continue_if_exception = {0}'.format(continue_if_exception))

133 timeout_override = actions[idx]['options'].pop('timeout_override')

134 logger.debug('timeout_override = {0}'.format(timeout_override))

135 ignore_empty_list = actions[idx]['options'].pop('ignore_empty_list')

136 logger.debug('ignore_empty_list = {0}'.format(ignore_empty_list))

137 allow_ilm = actions[idx]['options'].pop('allow_ilm_indices')

138 logger.debug('allow_ilm_indices = {0}'.format(allow_ilm))

139

140 ### Skip to next action if 'disabled'

141 if action_disabled:

142 logger.info(

143 'Action ID: {0}: "{1}" not performed because "disable_action" '

144 'is set to True'.format(idx, action)

145 )

146 continue

147 else:

148 logger.info('Preparing Action ID: {0}, "{1}"'.format(idx, action))

149 # Override the timeout, if specified, otherwise use the default.

150 if isinstance(timeout_override, int):

151 client_args['timeout'] = timeout_override

152 else:

153 client_args['timeout'] = default_timeout

154

155 # Set up action kwargs

156 kwargs = {}

157 kwargs['master_timeout'] = (

158 client_args['timeout'] if client_args['timeout'] <= 300 else 300)

159 kwargs['dry_run'] = dry_run

160

161 # Create a client object for each action...

162 logger.info('Creating client object and testing connection')

163 try:

164 client = get_client(**client_args)

165 except (ClientException, ConfigurationError):

166 sys.exit(1)

167 ### Filter ILM indices unless expressly permitted

168 if allow_ilm:

169 logger.warning('allow_ilm_indices: true')

170 logger.warning('Permitting operation on indices with an ILM policy')

171 if not allow_ilm and action not in settings.snapshot_actions():

172 if actions[idx]['action'] == 'rollover':

173 alias = actions[idx]['options']['name']

174 write_index = get_write_index(client, alias)

175 try:

176 idx_settings = client.indices.get_settings(index=write_index)

177 if 'name' in idx_settings[write_index]['settings']['index']['lifecycle']:

178 logger.info('Alias {0} is associated with ILM policy.'.format(alias))

179 logger.info(

180 'Skipping action {0} because allow_ilm_indices is false.'.format(idx))

181 continue

182 except KeyError:

183 logger.debug('No ILM policies associated with {0}'.format(alias))

184 elif 'filters' in actions[idx]:

185 actions[idx]['filters'].append({'filtertype': 'ilm'})

186 else:

187 actions[idx]['filters'] = [{'filtertype': 'ilm'}]

188 ##########################

189 ### Process the action ###

190 ##########################

191 try:

192 logger.info(

193 'Trying Action ID: {0}, "{1}": '

194 '{2}'.format(idx, action, actions[idx]['description'])

195 )

196 process_action(client, actions[idx], **kwargs)

197 except Exception as err:

198 if isinstance(err, NoIndices) or isinstance(err, NoSnapshots):

199 if ignore_empty_list:

200 logger.info(

201 'Skipping action "{0}" due to empty list: '

202 '{1}'.format(action, type(err))

203 )

204 else:

205 logger.error(

206 'Unable to complete action "{0}". No actionable items '

207 'in list: {1}'.format(action, type(err))

208 )

209 sys.exit(1)

210 else:

211 logger.error(

212 'Failed to complete action: {0}. {1}: '

213 '{2}'.format(action, type(err), err)

214 )

215 if continue_if_exception:

216 logger.info(

217 'Continuing execution with next action because '

218 '"continue_if_exception" is set to True for action '

219 '{0}'.format(action)

220 )

221 else:

222 sys.exit(1)

223 logger.info('Action ID: {0}, "{1}" completed.'.format(idx, action))

224 logger.info('Job completed.')

225

226@click.command()

227@click.option(

228 '--config',

229 help="Path to configuration file. Default: ~/.curator/curator.yml",

230 type=click.Path(exists=True), default=settings.config_file()

231)

232@click.option('--dry-run', is_flag=True, help='Do not perform any changes.')

233@click.argument('action_file', type=click.Path(exists=True), nargs=1)

234@click.version_option(version=__version__)

235 def cli(config, dry_run, action_file):

236 """

237 Curator for Elasticsearch indices.

238

239 See http://elastic.co/guide/en/elasticsearch/client/curator/current

240 """

241 run(config, action_file, dry_run)