Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/sqlite2mysql.py: 13%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/sqlite2mysql.py : 13%

## 104 statements 14 run 90 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4 Take a sqlite database file and copy the tables within it to a MySQL database

5

6 Usage:

7 sqlite2mysql -s <pathToSettingsFile> <pathToSqliteDB> [<tablePrefix>]

8

9 Options:

10

11 pathToSqliteDB path to the sqlite database file

12 tablePrefix a string to prefix the table names when copying to mysql database

13 pathToSettingsFile path to a settings file with logging and database information (yaml file)

14

15 -h, --help show this help message

16 -v, --version show version

17 -s, --settings the settings file

18"""

19################# GLOBAL IMPORTS ####################

20 from builtins import object

21 import sys

22 import os

23 import sqlite3 as lite

24 os.environ['TERM'] = 'vt100'

25 from fundamentals import tools

26 from fundamentals.mysql import writequery

27

28 from datetime import datetime, date, time

29

30

31 def main(arguments=None):

32 """

33 The main function used when ``yaml_to_database.py`` when installed as a cl tool

34 """

35

36 # setup the command-line util settings

37 su = tools(

38 arguments=arguments,

39 docString=__doc__,

40 logLevel="WARNING",

41 options_first=False,

42 projectName=False

43 )

44 arguments, settings, log, dbConn = su.setup()

45

46 # unpack remaining cl arguments using `exec` to setup the variable names

47 # automatically

48 for arg, val in list(arguments.items()):

49 if arg[0] == "-":

50 varname = arg.replace("-", "") + "Flag"

51 else:

52 varname = arg.replace("<", "").replace(">", "")

53 if isinstance(val, str):

54 exec(varname + " = '%s'" % (val,))

55 else:

56 exec(varname + " = %s" % (val,))

57 if arg == "--dbConn":

58 dbConn = val

59 log.debug('%s = %s' % (varname, val,))

60

61 from fundamentals.mysql import sqlite2mysql

62 converter = sqlite2mysql(

63 log=log,

64 settings=settings,

65 pathToSqlite=pathToSqliteDB,

66 tablePrefix=tablePrefix,

67 dbConn=dbConn

68 )

69 converter.convert_sqlite_to_mysql()

70

71 return

72

73

74 class sqlite2mysql(object):

75 """

76 *Take a sqlite database file and copy the tables within it to a MySQL database*

77

78 **Key Arguments:**

79 - ``log`` -- logger

80 - ``settings`` -- the settings dictionary

81 - ``pathToSqlite`` -- path to the sqlite database to transfer into the MySQL database

82 - ``tablePrefix`` -- a prefix to add to all the tablename when converting to mysql. Default *""*

83 - ``dbConn`` -- mysql database connection

84

85 **Usage:**

86

87 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).

88

89 To convert and import the content of a sqlite database into MySQL run the following:

90

91 .. todo::

92

93 - add a tutorial about ``sqlite2mysql`` to documentation

94

95 .. code-block:: python

96

97 from fundamentals.mysql import sqlite2mysql

98 converter = sqlite2mysql(

99 log=log,

100 settings=settings,

101 pathToSqlite="/path/to/sqlite.db",

102 tablePrefix="external"

103 )

104 converter.convert_sqlite_to_mysql()

105 """

106 # Initialisation

107

108 def __init__(

109 self,

110 log,

111 pathToSqlite,

112 tablePrefix="",

113 settings=False,

114 dbConn=False

115 ):

116 self.log = log

117 log.debug("instansiating a new 'sqlite2mysql' object")

118 self.settings = settings

119 self.pathToSqlite = pathToSqlite

120 self.tablePrefix = tablePrefix

121 self.dbConn = dbConn

122

123 if not self.tablePrefix:

124 self.tablePrefix = ""

125

126 if len(self.tablePrefix):

127 self.tablePrefix = self.tablePrefix + "_"

128 # xt-self-arg-tmpx

129

130 return None

131

132 def convert_sqlite_to_mysql(

133 self):

134 """*copy the contents of the sqlite database into the mysql database*

135

136 See class docstring for usage

137 """

138 from fundamentals.renderer import list_of_dictionaries

139 from fundamentals.mysql import directory_script_runner

140 self.log.debug('starting the ``convert_sqlite_to_mysql`` method')

141

142 con = lite.connect(self.pathToSqlite)

143 con.row_factory = lite.Row

144 cur = con.cursor()

145

146 # GET ALL TABLE NAMES

147 cur.execute("SELECT name FROM sqlite_master WHERE type='table';")

148 tables = cur.fetchall()

149

150 createStatements = []

151 inserts = []

152 for table in tables:

153 table = table['name']

154 if table == "sqlite_sequence":

155 continue

156

157 # CREATE TABLE collection_books (folder_id, fingerprint, primary key(folder_id, fingerprint));

158 # GENEREATE THE MYSQL CREATE STATEMENTS FOR EACH TABLE

159 cur.execute(

160 "SELECT sql FROM sqlite_master WHERE name = '%(table)s';" % locals())

161 createStatement = cur.fetchone()

162 createStatement = createStatement[0].replace('"', '`') + ";"

163 if "DEFAULT" not in createStatement:

164 if "primary key(" in createStatement:

165 tmp = createStatement.split("primary key(")

166 tmp[0] = tmp[0].replace(

167 ",", " varchar(150) DEFAULT NULL,")

168 createStatement = ("primary key(").join(tmp)

169 if "primary key," in createStatement:

170 tmp = createStatement.split("primary key,")

171 tmp[1] = tmp[1].replace(

172 ",", " varchar(150) DEFAULT NULL,")

173 tmp[1] = tmp[1].replace(

174 ");", " varchar(150) DEFAULT NULL);")

175 createStatement = ("primary key,").join(tmp)

176 createStatement = createStatement.replace(

177 "INTEGER PRIMARY KEY", "INTEGER AUTO_INCREMENT PRIMARY KEY")

178 createStatement = createStatement.replace(

179 "AUTOINCREMENT", "AUTO_INCREMENT")

180 createStatement = createStatement.replace(

181 "DEFAULT 't'", "DEFAULT '1'")

182 createStatement = createStatement.replace(

183 "DEFAULT 'f'", "DEFAULT '0'")

184 createStatement = createStatement.replace(",'t'", ",'1'")

185 createStatement = createStatement.replace(",'f'", ",'0'")

186 if "CREATE TABLE `" in createStatement:

187 createStatement = createStatement.replace(

188 "CREATE TABLE `", "CREATE TABLE IF NOT EXISTS `" + self.tablePrefix)

189 else:

190 createStatement = createStatement.replace(

191 "CREATE TABLE ", "CREATE TABLE IF NOT EXISTS " + self.tablePrefix)

192 if ", primary key(" in createStatement:

193 createStatement = createStatement.replace(", primary key(", """,

194`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,

195`dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,

196`updated` tinyint(4) DEFAULT '0',

197 primary key(""")

198 else:

199 createStatement = createStatement.replace(");", """,

200 `dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,

201 `dateLastModified` datetime DEFAULT CURRENT_TIMESTAMP,

202 `updated` tinyint(4) DEFAULT '0');

203 """)

204 createStatement = createStatement.replace(

205 " text primary key", " varchar(100) primary key")

206 createStatement = createStatement.replace(

207 "`EntryText` TEXT NOT NULL,", "`EntryText` TEXT,")

208 createStatement = createStatement.replace(

209 "`SelectionText` TEXT NOT NULL", "`SelectionText` TEXT")

210 createStatement = createStatement.replace(

211 "`Filename` INTEGER NOT NULL,", "`Filename` TEXT NOT NULL,")

212 createStatement = createStatement.replace(

213 "`SessionPartUUID` TEXT NOT NULL UNIQUE,", "`SessionPartUUID` VARCHAR(100) NOT NULL UNIQUE,")

214 createStatement = createStatement.replace(

215 "`Name` TEXT PRIMARY KEY NOT NULL", "`Name` VARCHAR(100) PRIMARY KEY NOT NULL")

216 createStatement = createStatement.replace(

217 " VARCHAR ", " VARCHAR(100) ")

218 createStatement = createStatement.replace(

219 " VARCHAR,", " VARCHAR(100),")

220

221 if len(createStatement.lower().split("datecreated")) > 2:

222 createStatement = createStatement.replace(

223 "`dateCreated` datetime DEFAULT CURRENT_TIMESTAMP,\n", "")

224

225 # GRAB THE DATA TO ADD TO THE MYSQL DATABASE TABLES

226 cur.execute(

227 "SELECT * from '%(table)s';" % locals())

228 rows = cur.fetchall()

229

230 allRows = []

231 for row in rows:

232 allRows.append(dict(row))

233

234 # RECURSIVELY CREATE MISSING DIRECTORIES

235 if not os.path.exists("/tmp/headjack/"):

236 os.makedirs("/tmp/headjack/")

237

238 writequery(

239 log=self.log,

240 sqlQuery=createStatement,

241 dbConn=self.dbConn,

242 )

243

244 from fundamentals.mysql import insert_list_of_dictionaries_into_database_tables

245 # USE dbSettings TO ACTIVATE MULTIPROCESSING

246 insert_list_of_dictionaries_into_database_tables(

247 dbConn=self.dbConn,

248 log=self.log,

249 dictList=allRows,

250 dbTableName=self.tablePrefix + table,

251 uniqueKeyList=[],

252 dateModified=True,

253 dateCreated=True,

254 batchSize=10000,

255 replace=True,

256 dbSettings=self.settings["database settings"]

257 )

258

259 # # DUMP THE DATA INTO A MYSQL DATABASE

260 # dataSet = list_of_dictionaries(

261 # log=self.log,

262 # listOfDictionaries=allRows

263 # )

264 # originalList = dataSet.list

265 # now = datetime.now()

266 # now = now.strftime("%Y%m%dt%H%M%S%f.sql")

267 # mysqlData = dataSet.mysql(

268 # tableName=self.tablePrefix + table, filepath="/tmp/headjack/" +

269 # now, createStatement=createStatement)

270

271 # directory_script_runner(

272 # log=self.log,

273 # pathToScriptDirectory="/tmp/headjack/",

274 # databaseName=self.settings["database settings"]["db"],

275 # loginPath=self.settings["database settings"]["loginPath"],

276 # successRule="delete",

277 # failureRule="failed"

278 # )

279

280 con.close()

281

282 self.log.debug('completed the ``convert_sqlite_to_mysql`` method')

283 return None

284

285 # use the tab-trigger below for new method

286 # xt-class-method

287

288 if __name__ == '__main__':

289 main()