Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/writequery.py: 7%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/writequery.py : 7%

## 115 statements 8 run 107 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Execute a MySQL write query on a database table*

5

6:Author:

7 David Young

8

9:Date Created:

10 June 21, 2016

11"""

12################# GLOBAL IMPORTS ####################

13 from builtins import str

14 import sys

15 import os

16 os.environ['TERM'] = 'vt100'

17 from fundamentals import tools

18 import time

19

20

21 def writequery(

22 log,

23 sqlQuery,

24 dbConn,

25 Force=False,

26 manyValueList=False

27):

28 """*Execute a MySQL write command given a sql query*

29

30 **Key Arguments:**

31 - ``sqlQuery`` -- the MySQL command to execute

32 - ``dbConn`` -- the db connection

33 - ``Force`` -- do not exit code if error occurs, move onto the next command

34 - ``manyValueList`` -- a list of value tuples if executing more than one insert

35

36 **Return:**

37 - ``message`` -- error/warning message

38

39 **Usage:**

40

41 Here's an example of how to create a table using the database connection passed to the function:

42

43 .. code-block:: python

44

45 from fundamentals.mysql import writequery

46 sqlQuery = "CREATE TABLE `testing_table` (`id` INT NOT NULL, PRIMARY KEY (`id`))"

47 message = writequery(

48 log=log,

49 sqlQuery=sqlQuery,

50 dbConn=dbConn,

51 Force=False,

52 manyValueList=False

53 )

54

55 Here's a many value insert example:

56

57 .. code-block:: python

58

59 from fundamentals.mysql import writequery

60 sqlQuery = "INSERT INTO testing_table (id) values (%s)"

61 message = writequery(

62 log=log,

63 sqlQuery=sqlQuery,

64 dbConn=dbConn,

65 Force=False,

66 manyValueList=[(1,), (2,), (3,), (4,), (5,), (6,), (7,),

67 (8,), (9,), (10,), (11,), (12,), ]

68 )

69

70 """

71 log.debug('starting the ``writequery`` function')

72 import pymysql

73 import warnings

74 warnings.filterwarnings('error', category=pymysql.Warning)

75 message = ""

76 try:

77 cursor = dbConn.cursor(pymysql.cursors.DictCursor)

78 except Exception as e:

79 log.error('could not create the database cursor.')

80 # EXECUTE THE SQL COMMAND

81

82 try:

83 if manyValueList == False:

84 cursor.execute(sqlQuery)

85

86 else:

87 # cursor.executemany(sqlQuery, manyValueList)

88 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER BARFING

89 batch = 100000

90 offset = 0

91 stop = 0

92

93 while stop == 0:

94 thisList = manyValueList[offset: offset + batch]

95 offset += batch

96 a = len(thisList)

97 cursor.executemany(sqlQuery, thisList)

98 dbConn.commit()

99 if len(thisList) < batch:

100 stop = 1

101 except pymysql.err.ProgrammingError as e:

102 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery,

103 str(e))

104 if Force == False:

105 log.error(message)

106 raise

107 else:

108 log.warning(message)

109 except pymysql.Error as e:

110

111 try:

112 e = e.args

113 except:

114 pass

115

116 if e[0] == 1050 and 'already exists' in e[1]:

117 log.info(str(e) + '\n')

118 elif e[0] == 1062:

119 # Duplicate Key error

120 log.debug('Duplicate Key error: %s\n' % (str(e), ))

121 message = "duplicate key error"

122 elif e[0] == 1061:

123 # Duplicate Key error

124 log.debug('index already exists: %s\n' % (str(e), ))

125 message = "index already exists"

126 elif "Duplicate entry" in str(e):

127 log.debug('Duplicate Key error: %s\n' % (str(e), ))

128 message = "duplicate key error"

129 elif "Deadlock" in str(e):

130 i = 0

131 while i < 10:

132 time.sleep(1)

133 i += 1

134 try:

135 if manyValueList == False:

136 cursor.execute(sqlQuery)

137 else:

138 # cursor.executemany(sqlQuery, manyValueList)

139 # INSET LARGE LISTS IN BATCHES TO STOP MYSQL SERVER

140 # BARFING

141 batch = 100000

142 offset = 0

143 stop = 0

144

145 while stop == 0:

146 thisList = manyValueList[offset: offset + batch]

147 offset += batch

148 a = len(thisList)

149 cursor.executemany(sqlQuery, thisList)

150 dbConn.commit()

151 if len(thisList) < batch:

152 stop = 1

153 i = 20

154 except:

155 pass

156 if i == 10:

157 log.error('Deadlock: %s\n' % (str(e), ))

158 message = "Deadlock error"

159 raise

160

161 else:

162 message = 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' % (sqlQuery,

163 str(e))

164 if Force == False:

165 log.error(message)

166 raise

167 else:

168 log.warning(message)

169

170 except pymysql.Warning as e:

171 log.info(str(e))

172 except Exception as e:

173 if "truncated" in str(e):

174 log.error('%s\n Here is the sqlquery:\n%s\n' % (str(e), sqlQuery))

175 if manyValueList:

176 log.error('... and the values:\n%s\n' % (thisList, ))

177 elif "Duplicate entry" in str(e):

178 log.warning('Duplicate Key error: %s\n' % (str(e), ))

179 message = "duplicate key error"

180 else:

181 log.error(

182 'MySQL write command not executed for this query: << %s >>\nThe error was: %s \n' %

183 (sqlQuery, str(e)))

184 if Force == False:

185 sys.exit(0)

186 cursor.close()

187 return -1

188 dbConn.commit()

189 # CLOSE THE CURSOR

190 cOpen = True

191 count = 0

192 while cOpen:

193 try:

194 cursor.close()

195 cOpen = False

196 except Exception as e:

197 time.sleep(1)

198 count += 1

199 if count == 10:

200 log.warning('could not close the db cursor ' + str(e) + '\n')

201 raise e

202 count = 0

203

204 log.debug('completed the ``writequery`` function')

205 return message