Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/yaml_to_database.py: 20%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/mysql/yaml_to_database.py : 20%

## 100 statements 20 run 80 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Take key-values from a yaml file including a tablename(s) and add them to a mysql database table*

5

6 Usage:

7 yaml2db [-d] -s <pathToSettingsFile> <pathToYaml>

8 yaml2db [-d] --host=<host> --user=<user> --passwd=<passwd> --dbName=<dbName> <pathToYaml>

9

10 Options:

11

12 pathToYaml path to a single yaml file or directory of yaml files

13 pathToSettingsFile path to a settings file with logging and database information (yaml file)

14 --host=<host> the database host

15 --user=<user> database user

16 --passwd=<passwd> database user password

17 --dbName=<dbName> name of the database to add the table content to

18

19 -d, --delete delete yaml open(s) once added to datbase

20 -h, --help show this help message

21 -v, --version show version

22 -s, --settings the settings file

23"""

24 from __future__ import print_function

25

26 from builtins import object

27 import sys

28 import os

29 import yaml

30 import requests

31 import re

32 import glob

33 import docopt

34 from fundamentals import tools, times

35 from fundamentals.mysql import convert_dictionary_to_mysql_table

36 import requests.packages.urllib3

37 requests.packages.urllib3.disable_warnings()

38

39

40 def main(arguments=None):

41 """

42 The main function used when ``yaml_to_database.py`` when installed as a cl tool

43 """

44

45 # setup the command-line util settings

46 su = tools(

47 arguments=arguments,

48 docString=__doc__,

49 logLevel="WARNING",

50 options_first=False,

51 projectName=False

52 )

53 arguments, settings, log, dbConn = su.setup()

54

55 # unpack remaining cl arguments using `exec` to setup the variable names

56 # automatically

57 for arg, val in list(arguments.items()):

58 if arg[0] == "-":

59 varname = arg.replace("-", "") + "Flag"

60 else:

61 varname = arg.replace("<", "").replace(">", "")

62 if isinstance(val, str):

63 exec(varname + " = '%s'" % (val,))

64 else:

65 exec(varname + " = %s" % (val,))

66 if arg == "--dbConn":

67 dbConn = val

68 log.debug('%s = %s' % (varname, val,))

69

70 if os.path.isfile(pathToYaml):

71 from fundamentals.mysql import yaml_to_database

72 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE

73 yaml2db = yaml_to_database(

74 log=log,

75 settings=settings,

76 dbConn=dbConn

77 )

78 yaml2db.add_yaml_file_content_to_database(

79 filepath=pathToYaml,

80 deleteFile=deleteFlag

81 )

82 basename = os.path.basename(pathToYaml)

83 print("Content of %(basename)s added to database" % locals())

84

85 else:

86 from fundamentals.mysql import yaml_to_database

87 yaml2db = yaml_to_database(

88 log=log,

89 settings=settings,

90 dbConn=dbConn,

91 pathToInputDir=pathToYaml,

92 deleteFiles=deleteFlag

93 )

94 yaml2db.ingest()

95 print("Content of %(pathToYaml)s directory added to database" % locals())

96

97 return

98

99

100 class yaml_to_database(object):

101 """

102 *Take key-values from yaml files including a tablename(s) and add them to a mysql database table*

103

104 **Key Arguments:**

105 - ``log`` -- logger

106 - ``settings`` -- the settings dictionary

107 - ``pathToInputDir`` -- path to the directory containing the yaml files that will be added to the database table(s). Default *False*

108 - ``dbConn`` -- connection to database to add the content to

109 - ``deleteFiles`` - - delete the yamls files once their content has been added to the database. Default * False*

110

111 **Usage:**

112

113 To setup your logger, settings and database connections, please use the ``fundamentals`` package (`see tutorial here <http://fundamentals.readthedocs.io/en/latest/#tutorial>`_).

114

115 To initiate a ``yaml2db`` object, use the following:

116

117 .. code-block:: python

118

119 from fundamentals.mysql import yaml_to_database

120 yaml2db = yaml_to_database(

121 log=log,

122 settings=settings,

123 dbConn=dbConn,

124 pathToInputDir="/path/to/yaml/directory",

125 deleteFiles=False

126 )

127

128 And here's an example of the content in a yaml file that this ``yaml2db`` object can parse:

129

130 .. code-block:: yaml

131

132 title: Why you should do most of your text editing in : Sublime Text | Sublime Text Tips

133 url: http://sublimetexttips.com/why-you-should-do-most-of-your-text-editing-in-sublime-text/?utm_source=drip&utm_medium=email&utm_campaign=editor-proliferation

134 kind: webpage

135 subtype: article

136 table: web_articles,podcasts

137 """

138 # Initialisation

139

140 def __init__(

141 self,

142 log,

143 dbConn,

144 pathToInputDir=False,

145 settings=False,

146 deleteFiles=False

147

148 ):

149 self.log = log

150 log.debug("instansiating a new 'yaml_to_database' object")

151 self.settings = settings

152 self.pathToInputDir = pathToInputDir

153 self.dbConn = dbConn

154 self.deleteFiles = deleteFiles

155 # xt-self-arg-tmpx

156

157 return None

158

159 def ingest(self):

160 """

161 *ingest the contents of the directory of yaml files into a database*

162

163 **Return:**

164 - None

165

166 **Usage:**

167

168 To import an entire directory of yaml files into a database, use the following:

169

170 .. code-block:: python

171

172 from fundamentals.mysql import yaml_to_database

173 yaml2db = yaml_to_database(

174 log=log,

175 settings=settings,

176 dbConn=dbConn,

177 pathToInputDir="/path/to/yaml/directory",

178 deleteFiles=False

179 )

180 yaml2db.ingest()

181 """

182 self.log.debug('starting the ``ingest`` method')

183

184 for d in os.listdir(self.pathToInputDir):

185 if os.path.isfile(os.path.join(self.pathToInputDir, d)) and "yaml" in d.lower():

186 self.add_yaml_file_content_to_database(

187 filepath=os.path.join(self.pathToInputDir, d),

188 deleteFile=self.deleteFiles

189 )

190

191 self.log.debug('completed the ``ingest`` method')

192 return None

193

194 def add_yaml_file_content_to_database(

195 self,

196 filepath,

197 deleteFile=False

198 ):

199 """*given a file to a yaml file, add yaml file content to database*

200

201 **Key Arguments:**

202 - ``filepath`` -- the path to the yaml file

203 - ``deleteFile`` -- delete the yaml file when its content has been added to the database. Default *False*

204

205 **Return:**

206 - None

207

208 **Usage:**

209

210 To parse and import the contents of a single yaml file into the database, use the following:

211

212 .. code-block:: python

213

214 from fundamentals.mysql import yaml_to_database

215 # PARSE YAML FILE CONTENTS AND ADD TO DATABASE

216 yaml2db = yaml_to_database(

217 log=log,

218 settings=settings,

219 dbConn=dbConn

220 )

221 yaml2db.add_yaml_file_content_to_database(

222 filepath=${1:"/path/to/file.yaml"},

223 deleteFile=True

224 )

225 """

226 self.log.debug(

227 'completed the ````add_yaml_file_content_to_database`` method')

228

229 import codecs

230 try:

231 self.log.debug("attempting to open the file %s" % (filepath,))

232 readFile = codecs.open(filepath, encoding='utf-8', mode='r')

233 thisData = readFile.read()

234 readFile.close()

235 except IOError as e:

236 message = 'could not open the file %s' % (filepath,)

237 self.log.critical(message)

238 raise IOError(message)

239 readFile.close()

240

241 matchObject = re.finditer(

242 r'(^|\n)(?P<key>[^\:]*)\:\s(?P<value>.*?)(\n|$)',

243 thisData,

244 flags=re.M | re.S # re.S

245 )

246

247 yamlContent = {}

248 for match in matchObject:

249 if match.group("value")[0] == '"' and match.group("value")[-1] == '"':

250 v = match.group("value")[1:-1]

251 elif match.group("value")[0] == "'" and match.group("value")[-1] == "'":

252 v = match.group("value")[1:-1]

253 else:

254 v = match.group("value")

255 yamlContent[match.group("key")] = v

256

257 if "table" not in yamlContent:

258 self.log.warning(

259 'A table value is need in the yaml content to indicate which database table to add the content to: %(filepath)s' % locals())

260 return None

261

262 # NOTE THERE MAY BE MORE THAN ONE DATABASE TABLE

263 dbTablesTmp = yamlContent["table"].split(",")

264 del yamlContent["table"]

265 dbTables = []

266 dbTables[:] = [d.strip() for d in dbTablesTmp]

267

268 # UNSHORTEN URL

269 try:

270 r = requests.head(yamlContent["url"], allow_redirects=True)

271 yamlContent["url"] = r.url

272 except:

273 pass

274

275 yamlContent["original_yaml_path"] = filepath

276

277 if "url" in yamlContent:

278 uniqueKeyList = ["url"]

279 else:

280 uniqueKeyList = []

281

282 for t in dbTables:

283 convert_dictionary_to_mysql_table(

284 dbConn=self.dbConn,

285 log=self.log,

286 dictionary=yamlContent,

287 dbTableName=t,

288 uniqueKeyList=uniqueKeyList,

289 dateModified=True,

290 returnInsertOnly=False,

291 replace=True

292 )

293 if deleteFile:

294 os.remove(filepath)

295

296 self.log.debug(

297 'completed the ``add_yaml_file_content_to_database`` method')

298 return None

299

300 # use the tab-trigger below for new method

301 # xt-class-method

302

303

304 if __name__ == '__main__':

305 main()