Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/renderer/list_of_dictionaries.py: 13%

# Coverage for /Users/Dave/git_repos/_packages_/python/fundamentals/fundamentals/renderer/list_of_dictionaries.py : 13%

## 223 statements 30 run 193 missing 0 excluded

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/local/bin/python

2# encoding: utf-8

3"""

4*Render a python list of dictionaries in various list and markup formats*

5

6:Author:

7 David Young

8

9:Date Created:

10 September 14, 2016

11"""

12################# GLOBAL IMPORTS ####################

13 from builtins import str

14 from builtins import range

15 from builtins import object

16 import sys

17 import os

18 import io

19 import re

20 import unicodecsv as csv

21 import codecs

22 import copy

23 import json

24 import yaml

25 from decimal import Decimal

26 from datetime import datetime

27 os.environ['TERM'] = 'vt100'

28 from fundamentals import tools

29 from fundamentals.mysql import convert_dictionary_to_mysql_table

30

31

32 class list_of_dictionaries(object):

33 """

34 *The dataset object is a list of python dictionaries. Using this class, the data can be rendered as various list and markup formats*

35

36 **Key Arguments:**

37 - ``log`` -- logger

38 - ``listOfDictionaries`` -- the list of dictionaries to render

39 - ``reDatetime`` -- a pre-compiled datetime regex. Default *False*fss

40

41 **Usage:**

42

43 To initialise the dataset object:

44

45 .. code-block:: python

46

47 dataList = [

48 {

49 "owner": "daisy",

50 "pet": "dog",

51 "address": "belfast, uk"

52 },

53 {

54 "owner": "john",

55 "pet": "snake",

56 "address": "the moon"

57 },

58 {

59 "owner": "susan",

60 "pet": "crocodile",

61 "address": "larne"

62 }

63

64 ]

65

66 from fundamentals.renderer import list_of_dictionaries

67 dataSet = list_of_dictionaries(

68 log=log,

69 listOfDictionaries=dataList

70 )

71 """

72

73 def __init__(

74 self,

75 log,

76 listOfDictionaries,

77 reDatetime=False

78 ):

79 self.log = log

80 self.log.debug("instansiating a new 'list_of_dictionaries' object")

81 self.listOfDictionaries = listOfDictionaries

82 self.reDatetime = reDatetime

83

84 return None

85

86 @property

87 def list(

88 self):

89 """*Returns the original list of dictionaries*

90

91 **Usage:**

92

93 dataSet.list

94 """

95 return self.listOfDictionaries

96

97 def csv(

98 self,

99 filepath=None

100 ):

101 """*Render the data in CSV format*

102

103 **Key Arguments:**

104 - ``filepath`` -- path to the file to write the csv content to. Default *None*

105

106 **Return:**

107 - ``renderedData`` -- the data rendered in csv format

108

109 **Usage:**

110

111 To render the data set as csv:

112

113 .. code-block:: python

114

115 print(dataSet.csv())

116

117 .. code-block:: text

118

119 owner,pet,address

120 daisy,dog,"belfast, uk"

121 john,snake,the moon

122 susan,crocodile,larne

123

124 and to save the csv rendering to file:

125

126 .. code-block:: python

127

128 dataSet.csv("/path/to/myfile.csv")

129 """

130 self.log.debug('starting the ``csv`` method')

131

132 renderedData = self._list_of_dictionaries_to_csv("machine")

133

134 if filepath and renderedData != "NO MATCH":

135

136 # RECURSIVELY CREATE MISSING DIRECTORIES

137 if not os.path.exists(os.path.dirname(filepath)):

138 os.makedirs(os.path.dirname(filepath))

139

140 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')

141 writeFile.write(renderedData)

142 writeFile.close()

143

144 self.log.debug('completed the ``csv`` method')

145 return renderedData

146

147 def table(

148 self,

149 filepath=None

150 ):

151 """*Render the data as a plain text table*

152

153 **Key Arguments:**

154 - ``filepath`` -- path to the file to write the table to. Default *None*

155

156 **Return:**

157 - ``renderedData`` -- the data rendered as a plain text table

158

159 **Usage:**

160

161 To render the data set as a plain text table:

162

163 .. code-block:: python

164

165 print(dataSet.table())

166

167 .. code-block:: text

168

169 +--------+------------+--------------+

170 | owner | pet | address |

171 +========+============+==============+

172 | daisy | dog | belfast, uk |

173 | john | snake | the moon |

174 | susan | crocodile | larne |

175 +--------+------------+--------------+

176

177 and to save the table rendering to file:

178

179 .. code-block:: python

180

181 dataSet.table("/path/to/myfile.ascii")

182 """

183 self.log.debug('starting the ``table`` method')

184

185 self.filepath = filepath

186 renderedData = self._list_of_dictionaries_to_csv("human")

187

188 if filepath and len(self.listOfDictionaries):

189

190 # RECURSIVELY CREATE MISSING DIRECTORIES

191 if not os.path.exists(os.path.dirname(filepath)):

192 os.makedirs(os.path.dirname(filepath))

193

194 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')

195 writeFile.write(renderedData)

196 writeFile.close()

197

198 self.log.debug('completed the ``table`` method')

199 return renderedData

200

201 def reST(

202 self,

203 filepath=None

204 ):

205 """*Render the data as a resturcturedText table*

206

207 **Key Arguments:**

208 - ``filepath`` -- path to the file to write the table to. Default *None*

209

210 **Return:**

211 - ``renderedData`` -- the data rendered as a resturcturedText table

212

213 **Usage:**

214

215 To render the data set as a resturcturedText table:

216

217 .. code-block:: python

218

219 print(dataSet.reST())

220

221 .. code-block:: text

222

223 +--------+------------+--------------+

224 | owner | pet | address |

225 +========+============+==============+

226 | daisy | dog | belfast, uk |

227 +--------+------------+--------------+

228 | john | snake | the moon |

229 +--------+------------+--------------+

230 | susan | crocodile | larne |

231 +--------+------------+--------------+

232

233 and to save the table rendering to file:

234

235 .. code-block:: python

236

237 dataSet.reST("/path/to/myfile.rst")

238 """

239 self.log.debug('starting the ``table`` method')

240

241 self.filepath = filepath

242 renderedData = self._list_of_dictionaries_to_csv("reST")

243

244 if filepath and len(self.listOfDictionaries):

245

246 # RECURSIVELY CREATE MISSING DIRECTORIES

247 if not os.path.exists(os.path.dirname(filepath)):

248 os.makedirs(os.path.dirname(filepath))

249

250 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')

251 writeFile.write(renderedData)

252 writeFile.close()

253

254 self.log.debug('completed the ``table`` method')

255 return renderedData

256

257 def markdown(

258 self,

259 filepath=None

260 ):

261 """*Render the data as a markdown table*

262

263 **Key Arguments:**

264 - ``filepath`` -- path to the file to write the markdown to. Default *None*

265

266 **Return:**

267 - ``renderedData`` -- the data rendered as a markdown table

268

269 **Usage:**

270

271 To render the data set as a markdown table:

272

273 .. code-block:: python

274

275 print(dataSet.markdown())

276

277 .. code-block:: markdown

278

279 | owner | pet | address |

280 |:-------|:-----------|:-------------|

281 | daisy | dog | belfast, uk |

282 | john | snake | the moon |

283 | susan | crocodile | larne |

284

285 and to save the markdown table rendering to file:

286

287 .. code-block:: python

288

289 dataSet.table("/path/to/myfile.md")

290 """

291 self.log.debug('starting the ``markdown`` method')

292

293 self.filepath = filepath

294 renderedData = self._list_of_dictionaries_to_csv("markdown")

295

296 if filepath and len(self.listOfDictionaries):

297

298 # RECURSIVELY CREATE MISSING DIRECTORIES

299 if not os.path.exists(os.path.dirname(filepath)):

300 os.makedirs(os.path.dirname(filepath))

301

302 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')

303 writeFile.write(renderedData)

304 writeFile.close()

305

306 self.log.debug('completed the ``markdown`` method')

307 return renderedData

308

309 def json(

310 self,

311 filepath=None

312 ):

313 """*Render the data in json format*

314

315 **Key Arguments:**

316 - ``filepath`` -- path to the file to write the json content to. Default *None*

317

318 **Return:**

319 - ``renderedData`` -- the data rendered as json

320

321 **Usage:**

322

323 To render the data set as json:

324

325 .. code-block:: python

326

327 print(dataSet.json())

328

329 .. code-block:: json

330

331 [

332 {

333 "address": "belfast, uk",

334 "owner": "daisy",

335 "pet": "dog"

336 },

337 {

338 "address": "the moon",

339 "owner": "john",

340 "pet": "snake"

341 },

342 {

343 "address": "larne",

344 "owner": "susan",

345 "pet": "crocodile"

346 }

347 ]

348

349 and to save the json rendering to file:

350

351 .. code-block:: python

352

353 dataSet.json("/path/to/myfile.json")

354 """

355 self.log.debug('starting the ``json`` method')

356

357 dataCopy = copy.deepcopy(self.listOfDictionaries)

358 for d in dataCopy:

359 for k, v in list(d.items()):

360 if isinstance(v, datetime):

361 d[k] = v.strftime("%Y%m%dt%H%M%S")

362

363 renderedData = json.dumps(

364 dataCopy,

365 separators=(',', ': '),

366 sort_keys=True,

367 indent=4

368 )

369

370 if filepath and len(self.listOfDictionaries):

371

372 # RECURSIVELY CREATE MISSING DIRECTORIES

373 if not os.path.exists(os.path.dirname(filepath)):

374 os.makedirs(os.path.dirname(filepath))

375

376 writeFile = codecs.open(filepath, encoding='utf-8', mode='w')

377 writeFile.write(renderedData)

378 writeFile.close()

379

380 self.log.debug('completed the ``json`` method')

381 return renderedData

382

383 def yaml(

384 self,

385 filepath=None

386 ):

387 """*Render the data in yaml format*

388

389 **Key Arguments:**

390 - ``filepath`` -- path to the file to write the yaml content to. Default *None*

391

392 **Return:**

393 - ``renderedData`` -- the data rendered as yaml

394

395 **Usage:**

396

397 To render the data set as yaml:

398

399 .. code-block:: python

400

401 print(dataSet.yaml())

402

403 .. code-block:: yaml

404

405 - address: belfast, uk

406 owner: daisy

407 pet: dog

408 - address: the moon

409 owner: john

410 pet: snake

411 - address: larne

412 owner: susan

413 pet: crocodile

414

415 and to save the yaml rendering to file:

416

417 .. code-block:: python

418

419 dataSet.json("/path/to/myfile.yaml")

420 """

421 self.log.debug('starting the ``yaml`` method')

422

423 dataCopy = []

424 dataCopy[:] = [dict(l) for l in self.listOfDictionaries]

425 renderedData = yaml.dump(dataCopy, default_flow_style=False)

426

427 if filepath and len(self.listOfDictionaries):

428

429 # RECURSIVELY CREATE MISSING DIRECTORIES

430 if not os.path.exists(os.path.dirname(filepath)):

431 os.makedirs(os.path.dirname(filepath))

432

433 stream = open(filepath, 'w')

434 yaml.dump(dataCopy, stream, default_flow_style=False)

435 stream.close()

436

437 self.log.debug('completed the ``yaml`` method')

438 return renderedData

439

440 def mysql(

441 self,

442 tableName,

443 filepath=None,

444 createStatement=None

445 ):

446 """*Render the dataset as a series of mysql insert statements*

447

448 **Key Arguments:**

449 - ``tableName`` -- the name of the mysql db table to assign the insert statements to.

450 - ``filepath`` -- path to the file to write the mysql inserts content to. Default *None*

451 createStatement

452

453 **Return:**

454 - ``renderedData`` -- the data rendered mysql insert statements (string format)

455

456 **Usage:**

457

458 .. code-block:: python

459

460 print(dataSet.mysql("testing_table"))

461

462 this output the following:

463

464 .. code-block:: plain

465

466 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("belfast, uk" ,"2016-09-14T16:21:36" ,"daisy" ,"dog") ON DUPLICATE KEY UPDATE address="belfast, uk", dateCreated="2016-09-14T16:21:36", owner="daisy", pet="dog" ;

467 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("the moon" ,"2016-09-14T16:21:36" ,"john" ,"snake") ON DUPLICATE KEY UPDATE address="the moon", dateCreated="2016-09-14T16:21:36", owner="john", pet="snake" ;

468 INSERT INTO `testing_table` (address,dateCreated,owner,pet) VALUES ("larne" ,"2016-09-14T16:21:36" ,"susan" ,"crocodile") ON DUPLICATE KEY UPDATE address="larne", dateCreated="2016-09-14T16:21:36", owner="susan", pet="crocodile" ;

469

470 To save this rendering to file use:

471

472 .. code-block:: python

473

474 dataSet.mysql("testing_table", "/path/to/myfile.sql")

475

476 """

477 self.log.debug('starting the ``mysql`` method')

478

479 import re

480 if createStatement and "create table if not exists" not in createStatement.lower():

481 regex = re.compile(r'^\s*CREATE TABLE ', re.I | re.S)

482 createStatement = regex.sub(

483 "CREATE TABLE IF NOT EXISTS ", createStatement)

484

485 renderedData = self._list_of_dictionaries_to_mysql_inserts(

486 tableName=tableName,

487 createStatement=createStatement

488 )

489

490 if filepath and len(self.listOfDictionaries):

491

492 # RECURSIVELY CREATE MISSING DIRECTORIES

493 if not os.path.exists(os.path.dirname(filepath)):

494 os.makedirs(os.path.dirname(filepath))

495

496 writeFile = open(filepath, mode='w')

497 writeFile.write(renderedData)

498 writeFile.close()

499

500 self.log.debug('completed the ``mysql`` method')

501 return renderedData

502

503 def _list_of_dictionaries_to_csv(

504 self,

505 csvType="human"):

506 """Convert a python list of dictionaries to pretty csv output

507

508 **Key Arguments:**

509 - ``csvType`` -- human, machine or reST

510

511 **Return:**

512 - ``output`` -- the contents of a CSV file

513 """

514 self.log.debug(

515 'starting the ``_list_of_dictionaries_to_csv`` function')

516

517 if not len(self.listOfDictionaries):

518 return "NO MATCH"

519

520 dataCopy = copy.deepcopy(self.listOfDictionaries)

521

522 tableColumnNames = list(dataCopy[0].keys())

523 columnWidths = []

524 columnWidths[:] = [len(tableColumnNames[i])

525 for i in range(len(tableColumnNames))]

526

527 output = io.BytesIO()

528 # setup csv styles

529 if csvType == "machine":

530 delimiter = ","

531 elif csvType in ["human", "markdown"]:

532 delimiter = "|"

533 elif csvType in ["reST"]:

534 delimiter = "|"

535 if csvType in ["markdown"]:

536 writer = csv.writer(output, delimiter=delimiter,

537 quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n")

538 else:

539 writer = csv.writer(output, dialect='excel', delimiter=delimiter,

540 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n")

541

542 if csvType in ["markdown"]:

543 dividerWriter = csv.writer(

544 output, delimiter="|", quoting=csv.QUOTE_NONE, doublequote=False, quotechar='"', escapechar="\\", lineterminator="\n")

545 else:

546 dividerWriter = csv.writer(output, dialect='excel', delimiter="+",

547 quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator="\n")

548 # add column names to csv

549 header = []

550 divider = []

551 rstDivider = []

552 allRows = []

553

554 # clean up data

555 for row in dataCopy:

556 for c in tableColumnNames:

557 if isinstance(row[c], float) or isinstance(row[c], Decimal):

558 row[c] = "%0.9g" % row[c]

559 elif isinstance(row[c], datetime):

560 thisDate = str(row[c])[:10]

561 row[c] = "%(thisDate)s" % locals()

562

563 # set the column widths

564 for row in dataCopy:

565 for i, c in enumerate(tableColumnNames):

566 if len(str(row[c])) > columnWidths[i]:

567 columnWidths[i] = len(str(row[c]))

568

569 # table borders for human readable

570 if csvType in ["human", "markdown", "reST"]:

571 header.append("")

572 divider.append("")

573 rstDivider.append("")

574

575 for i, c in enumerate(tableColumnNames):

576 if csvType == "machine":

577 header.append(c)

578 elif csvType in ["human", "markdown", "reST"]:

579 header.append(

580 c.ljust(columnWidths[i] + 2).rjust(columnWidths[i] + 3))

581 divider.append('-' * (columnWidths[i] + 3))

582 rstDivider.append('=' * (columnWidths[i] + 3))

583

584 # table border for human readable

585 if csvType in ["human", "markdown", "reST"]:

586 header.append("")

587 divider.append("")

588 rstDivider.append("")

589

590 # fill in the data

591 for row in dataCopy:

592 thisRow = []

593 # table border for human readable

594 if csvType in ["human", "markdown", "reST"]:

595 thisRow.append("")

596

597 for i, c in enumerate(tableColumnNames):

598 if csvType in ["human", "markdown", "reST"]:

599 if row[c] == None:

600 row[c] = ""

601 row[c] = str(str(row[c]).ljust(columnWidths[i] + 2)

602 .rjust(columnWidths[i] + 3))

603 thisRow.append(row[c])

604 # table border for human readable

605 if csvType in ["human", "markdown", "reST"]:

606 thisRow.append("")

607 allRows.append(thisRow)

608 if csvType in ["reST"]:

609 allRows.append(divider)

610

611 if csvType == "machine":

612 writer.writerow(header)

613 if csvType in ["reST"]:

614 dividerWriter.writerow(divider)

615 writer.writerow(header)

616 dividerWriter.writerow(rstDivider)

617 if csvType in ["human"]:

618 dividerWriter.writerow(divider)

619 writer.writerow(header)

620 dividerWriter.writerow(divider)

621 elif csvType in ["markdown"]:

622 writer.writerow(header)

623 dividerWriter.writerow(divider)

624

625 # write out the data

626 writer.writerows(allRows)

627 # table border for human readable

628 if csvType in ["human"]:

629 dividerWriter.writerow(divider)

630

631 output = output.getvalue()

632 output = output.strip()

633 output = str(output)

634

635 if csvType in ["markdown"]:

636 output = output.replace("|--", "|:-")

637 if csvType in ["reST"]:

638 output = output.replace("|--", "+--").replace("--|", "--+")

639

640 self.log.debug(

641 'completed the ``_list_of_dictionaries_to_csv`` function')

642

643 return output

644

645 def _list_of_dictionaries_to_mysql_inserts(

646 self,

647 tableName,

648 createStatement=None):

649 """Convert a python list of dictionaries to pretty csv output

650

651 **Key Arguments:**

652 - ``tableName`` -- the name of the table to create the insert statements for

653 - ``createStatement`` -- add this create statement to the top of the file. Will only be executed if no table of that name exists in database. Default *None*

654

655 **Return:**

656 - ``output`` -- the mysql insert statements (as a string)

657 """

658 self.log.debug(

659 'completed the ````_list_of_dictionaries_to_mysql_inserts`` function')

660

661 if not len(self.listOfDictionaries):

662 return "NO MATCH"

663

664 dataCopy = copy.deepcopy(self.listOfDictionaries)

665

666 if createStatement:

667 output = createStatement + "\n"

668 else:

669 output = ""

670

671 inserts = []

672

673 inserts = []

674 inserts[:] = [convert_dictionary_to_mysql_table(log=self.log, dictionary=d, dbTableName=tableName, uniqueKeyList=[

675 ], dateModified=False, returnInsertOnly=True, replace=True, batchInserts=False, reDatetime=self.reDatetime) for d in dataCopy]

676 output += ";\n".join(inserts) + ";"

677

678 self.log.debug(

679 'completed the ``_list_of_dictionaries_to_mysql_inserts`` function')

680 return output