Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/directory_script_runner.py: 15%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/directory_script_runner.py : 15%

## 87 statements 13 run 74 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Given a directory of MySQL scripts, execute the scripts and process the script files according to their success or failure*

5

6 Run the following code once to set a login-path for your mysql server:

7

8 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password

9

10 This store's your credentials in an encrypted file located at '~/.mylogin.cnf'.

11 Use `mysql_config_editor print --all` to see all of the login-paths set.

12

13

14 Usage:

15 mysqlSucker <pathToDirectory> <loginPath> <databaseName> [-s successRule -f failureRule]

16

17 pathToDirectory path to the directory containing the sql scripts to run (scripts must have `.sql` extension)

18 loginPath the local-path as set with `mysql_config_editor` (`mysqlSucker -h` for more details)

19 databaseName the name of the database to execute the scripts within

20

21 Options:

22 -h, --help show this help message

23 -s successRule, --success successRule what to do if script succeeds. Default *None* [None|delete|subFolderName]

24 -f failureRule, --failure failureRule what to do if script fails. Default *None* [None|delete|subFolderName]

25

26:Examples:

27

28 To simply execute the scripts in a directory you can run:

29

30 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName

31

32 To delete script after thay have executed successfully:

33

34 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s delete

35

36 To move successful script to a `passed` sub-directory of `/path/to/scriptdir` and failed scripts to a `failed` sub-directory

37

38 mysqlSucker /path/to/scriptdir myLoginPath myDatabaseName -s pass -f failed

39

40:Author:

41 David Young

42

43:Date Created:

44 September 22, 2016

45"""

46################# GLOBAL IMPORTS ####################

47 from builtins import str

48 import sys

49 import os

50 import time

51 import collections

52 import datetime

53 from subprocess import Popen, PIPE, STDOUT

54 os.environ['TERM'] = 'vt100'

55 from fundamentals import tools

56

57

58 def directory_script_runner(

59 log,

60 pathToScriptDirectory,

61 databaseName,

62 loginPath,

63 force=True,

64 waitForResult=True,

65 successRule=None,

66 failureRule=None):

67 """A function to run all of the mysql scripts in a given directory (in a modified date order, oldest first) and then act on the script file in accordance with the succcess or failure of its execution

68

69 As it's insecure to pass in mysql database credentials via the command-line, run the following command from the terminal

70

71 .. code-block:: bash

72

73 mysql_config_editor set --login-path=<uniqueLoginName> --host=localhost --user=<myUsername> --password

74 > Enter password:

75

76 This will store your credentials in an encrypted file located at '~/.mylogin.cnf'. This function takes advantage of the `--login-path` so as not to compromise the user's credentials. Use `mysql_config_editor print --all` to see all of the login-paths set.

77

78 **Key Arguments:**

79 - ``log`` -- logger

80 - ``pathToScriptDirectory`` -- the path to the directory containing the sql script to be run

81 - ``databaseName`` -- the name of the database

82 - ``force`` -- force the script to run, skipping over lines with errors, Default *True*

83 - ``loginPath`` -- the local-path as set with `mysql_config_editor`

84 - ``waitForResult`` -- wait for the mysql script to finish execution? If 'False' the MySQL script will run in background (do not wait for completion), or if 'delete' the script will run then delete regardless of success status. Default *True*. [True|False|delete]

85 - ``successRule`` -- what to do if script succeeds. Default *None* [None|delete|subFolderName]

86 - ``failureRule`` -- what to do if script fails. Default *None* [None|delete|subFolderName]

87

88 **Return:**

89 - None

90

91 **Usage:**

92

93 To run the scripts in the directroy and not act on the script file use something similar to:

94

95 .. code-block:: python

96

97 from fundamentals.mysql import directory_script_runner

98 directory_script_runner(

99 log=log,

100 pathToScriptDirectory="/path/to/mysql_scripts",

101 databaseName="imports",

102 loginPath="myLoginDetails"

103 )

104

105 To delete successful scripts and archive failed scripts for later inspection:

106

107 .. code-block:: python

108

109 from fundamentals.mysql import directory_script_runner

110 directory_script_runner(

111 log=log,

112 pathToScriptDirectory="/path/to/mysql_scripts",

113 databaseName="imports",

114 loginPath="myLoginDetails",

115 successRule="delete",

116 failureRule="failed"

117 )

118

119 This creates a folder at `/path/to/mysql_scripts/failed` and moves the failed scripts into that folder.

120

121 Finally to execute the scripts within a directory but not wait for the results to return (much fast but you lose error checking in the MySQL scripts):

122

123 .. code-block:: python

124

125 from fundamentals.mysql import directory_script_runner

126 directory_script_runner(

127 log=log,

128 pathToScriptDirectory="/path/to/mysql_scripts",

129 databaseName="imports",

130 loginPath="myLoginDetails",

131 waitForResults=False

132 )

133

134 Setting ``waitForResults`` = 'delete' will trash the script once it has run (or failed ... be very careful!)

135 """

136 log.debug('starting the ``directory_script_runner`` function')

137

138 # COMPILE A DICTIONARY OF SCRIPTS / MODIFIED TIMES

139 scriptList = {}

140 for d in os.listdir(pathToScriptDirectory):

141 filePath = os.path.join(pathToScriptDirectory, d)

142 filename = os.path.basename(filePath)

143 extension = filePath.split(".")[-1]

144 if os.path.isfile(filePath) and extension == "sql":

145 modified = datetime.datetime.strptime(time.ctime(

146 os.path.getmtime(filePath)), "%a %b %d %H:%M:%S %Y")

147 scriptList[str(modified) + filename] = filePath

148

149 # FORCE MYSQL SCRIPT?

150 if force:

151 force = "--force"

152 else:

153 force = ""

154

155 # ORDER THE DICTIONARY BY MODIFIED TIME - OLDEST FIRST

156 scriptList = collections.OrderedDict(sorted(scriptList.items()))

157 for k, v in list(scriptList.items()):

158 scriptname = os.path.basename(v)

159 if waitForResult == True:

160 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" """ % locals(

161 )

162 p = Popen(cmd, stdout=PIPE, stderr=PIPE, close_fds=True,

163 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:/usr/bin:/usr/local/mysql/bin", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True)

164 stdout, stderr = p.communicate()

165

166 if len(stderr):

167 log.error(

168 "MySQL Script `%(scriptname)s` Failed: '%(stderr)s'" % locals())

169 if failureRule == None or failureRule == False:

170 pass

171 elif failureRule == "delete":

172 os.remove(v)

173 elif "/" not in failureRule:

174 moveTo = pathToScriptDirectory + "/" + failureRule

175 # Recursively create missing directories

176 if not os.path.exists(moveTo):

177 os.makedirs(moveTo)

178 moveTo = moveTo + "/" + scriptname

179 try:

180 log.debug("attempting to rename file %s to %s" %

181 (v, moveTo))

182 os.rename(v, moveTo)

183 except Exception as e:

184 log.error(

185 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),))

186 else:

187 if successRule == None or successRule == False:

188 pass

189 elif successRule == "delete":

190 os.remove(v)

191 elif "/" not in successRule:

192 moveTo = pathToScriptDirectory + "/" + successRule

193 # Recursively create missing directories

194 if not os.path.exists(moveTo):

195 os.makedirs(moveTo)

196 moveTo = moveTo + "/" + scriptname

197 try:

198 log.debug("attempting to rename file %s to %s" %

199 (v, moveTo))

200 os.rename(v, moveTo)

201 except Exception as e:

202 log.error(

203 "could not rename file %s to %s - failed with this error: %s " % (v, moveTo, str(e),))

204 else:

205 if waitForResult == "delete":

206 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 & rm "%(v)s" """ % locals()

207 else:

208 cmd = """mysql --login-path=%(loginPath)s %(force)s %(databaseName)s < "%(v)s" > /dev/null 2>&1 """ % locals()

209 p = Popen(cmd, close_fds=True,

210 env={'PATH': os.getenv('PATH') + ":/usr/local/bin:/usr/bin:", "MYSQL_TEST_LOGIN_FILE": os.getenv('HOME') + "/.mylogin.cnf"}, shell=True, stdin=None, stdout=None, stderr=None)

211

212 log.debug('completed the ``directory_script_runner`` function')

213 return None

214

215

216 def main(arguments=None):

217 """

218 The main function used when ``directory_script_runner.py`` is run as a single script from the cl, or when installed as a cl command

219 """

220

221 # setup the command-line util settings

222 su = tools(

223 arguments=arguments,

224 docString=__doc__,

225 logLevel="WARNING",

226 options_first=False,

227 projectName="fundmentals"

228 )

229 arguments, settings, log, dbConn = su.setup()

230

231 # UNPACK REMAINING CL ARGUMENTS USING `EXEC` TO SETUP THE VARIABLE NAMES

232 # AUTOMATICALLY

233 for arg, val in list(arguments.items()):

234 if arg[0] == "-":

235 varname = arg.replace("-", "") + "Flag"

236 else:

237 varname = arg.replace("<", "").replace(">", "")

238 if isinstance(val, ("".__class__, u"".__class__)):

239 exec(varname + " = '%s'" % (val,))

240 else:

241 exec(varname + " = %s" % (val,))

242 if arg == "--dbConn":

243 dbConn = val

244 log.debug('%s = %s' % (varname, val,))

245

246 if successFlag and successFlag.lower() == "none":

247 successFlag = None

248 if failureFlag and failureFlag.lower() == "none":

249 failureFlag = None

250

251 directory_script_runner(

252 log=log,

253 pathToScriptDirectory=pathToDirectory,

254 databaseName=databaseName,

255 loginPath=loginPath,

256 successRule=successFlag,

257 failureRule=failureFlag

258 )

259

260 return

261

262 if __name__ == '__main__':

263 main()