Coverage for jutil/format.py: 82%

# Coverage for jutil/format.py : 82%

## 268 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import csv

2 import html

3 import json

4 import logging

5 import os

6 import re

7 import tempfile

8 from collections import OrderedDict

9 from datetime import timedelta

10 from decimal import Decimal

11 import subprocess

12 from io import StringIO

13 from typing import List, Any, Optional, Union, Dict, Sequence, Tuple, TypeVar

14 from django.conf import settings

15 from django.core.exceptions import ValidationError

16 from django.utils.functional import lazy

17 import xml.dom.minidom # type: ignore

18 from django.utils.safestring import mark_safe

19 from django.utils.text import capfirst

20

21 logger = logging.getLogger(__name__)

22

23 S = TypeVar("S")

24

25

26 def format_full_name(first_name: str, last_name: str, max_length: int = 20) -> str:

27 """

28 Limits name length to specified length. Tries to keep name as human-readable an natural as possible.

29 :param first_name: First name

30 :param last_name: Last name

31 :param max_length: Maximum length

32 :return: Full name of shortened version depending on length

33 """

34 # dont allow commas in limited names

35 first_name = first_name.replace(",", " ")

36 last_name = last_name.replace(",", " ")

37

38 # accept short full names as is

39 original_full_name = first_name + " " + last_name

40 if len(original_full_name) <= max_length:

41 return original_full_name

42

43 # drop middle names

44 first_name = first_name.split(" ")[0]

45 full_name = first_name + " " + last_name

46 if len(full_name) <= max_length:

47 return full_name

48

49 # drop latter parts of combined first names

50 first_name = re.split(r"[\s\-]", first_name)[0]

51 full_name = first_name + " " + last_name

52 if len(full_name) <= max_length:

53 return full_name

54

55 # drop latter parts of multi part last names

56 last_name = re.split(r"[\s\-]", last_name)[0]

57 full_name = first_name + " " + last_name

58 if len(full_name) <= max_length:

59 return full_name

60

61 # shorten last name to one letter

62 last_name = last_name[:1]

63

64 full_name = first_name + " " + last_name

65 if len(full_name) > max_length:

66 raise Exception("Failed to shorten name {}".format(original_full_name))

67 return full_name

68

69

70 def format_timedelta(

71 dt: timedelta, days_label: str = "d", hours_label: str = "h", minutes_label: str = "min", seconds_label: str = "s"

72) -> str:

73 """

74 Formats timedelta to readable format, e.g. 1h30min15s.

75 :param dt: timedelta

76 :param days_label: Label for days. Leave empty '' if value should be skipped / ignored.

77 :param hours_label: Label for hours. Leave empty '' if value should be skipped / ignored.

78 :param minutes_label: Label for minutes. Leave empty '' if value should be skipped / ignored.

79 :param seconds_label: Label for seconds. Leave empty '' if value should be skipped / ignored.

80 :return: str

81 """

82 parts = (

83 (86400, days_label),

84 (3600, hours_label),

85 (60, minutes_label),

86 (1, seconds_label),

87 )

88 out = ""

89 seconds_f = dt.total_seconds()

90 seconds = int(seconds_f)

91 for n_secs, label in parts:

92 n, remainder = divmod(seconds, n_secs)

93 if n > 0 and label:

94 out += str(n) + label

95 seconds = remainder

96 out_str = out.strip()

97 if not out_str:

98 if seconds_f >= 0.001: 98 ↛ 101line 98 didn't jump to line 101, because the condition on line 98 was never false

99 out_str = "{:0.3f}".format(int(seconds_f * 1000.0) * 0.001) + seconds_label

100 else:

101 out_str = "0" + seconds_label

102 return out_str.strip()

103

104

105 def format_xml(content: str, encoding: str = "UTF-8", exceptions: bool = False) -> str:

106 """

107 Formats XML document as human-readable plain text.

108 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

109 :param content: XML data as str

110 :param encoding: XML file encoding

111 :param exceptions: Raise exceptions on error

112 :return: str (Formatted XML str)

113 """

114 assert isinstance(content, str)

115 try:

116 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 116 ↛ 122line 116 didn't jump to line 122, because the condition on line 116 was never false

117 with tempfile.NamedTemporaryFile() as fp:

118 fp.write(content.encode(encoding=encoding))

119 fp.flush()

120 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name])

121 return out.decode(encoding=encoding)

122 return xml.dom.minidom.parseString(content).toprettyxml()

123 except Exception as e:

124 logger.error("format_xml failed: %s", e)

125 if exceptions:

126 raise

127 return content

128

129

130 def format_xml_bytes(content: bytes, encoding: str = "UTF-8", exceptions: bool = False) -> bytes:

131 """

132 Formats XML document as human-readable plain text and returns result in bytes.

133 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

134 :param content: XML data as bytes

135 :param encoding: XML file encoding

136 :param exceptions: Raise exceptions on error

137 :return: bytes (Formatted XML as bytes)

138 """

139 assert isinstance(content, bytes)

140 try:

141 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH: 141 ↛ 147line 141 didn't jump to line 147, because the condition on line 141 was never false

142 with tempfile.NamedTemporaryFile() as fp:

143 fp.write(content)

144 fp.flush()

145 out = subprocess.check_output([settings.XMLLINT_PATH, "--format", fp.name])

146 return out

147 return xml.dom.minidom.parseString(content.decode(encoding=encoding)).toprettyxml(encoding=encoding)

148 except Exception as e:

149 logger.error("format_xml_bytes failed: %s", e)

150 if exceptions:

151 raise

152 return content

153

154

155 def format_xml_file(full_path: str, encoding: str = "UTF-8", exceptions: bool = False) -> bytes:

156 """

157 Formats XML file as human-readable plain text and returns result in bytes.

158 Tries to format XML file first, if formatting fails the file content is returned as is.

159 If the file does not exist empty bytes is returned.

160 If settings.XMLLINT_PATH is defined xmllint is used for formatting (higher quality). Otherwise minidom toprettyxml is used.

161 :param full_path: Full path to XML file

162 :param encoding: XML file encoding

163 :param exceptions: Raise exceptions on error

164 :return: bytes

165 """

166 try:

167 if hasattr(settings, "XMLLINT_PATH") and settings.XMLLINT_PATH:

168 return subprocess.check_output([settings.XMLLINT_PATH, "--format", full_path])

169 with open(full_path, "rb") as fp:

170 return xml.dom.minidom.parse(fp).toprettyxml(encoding=encoding) # type: ignore

171 except Exception as e:

172 logger.error("format_xml_file failed (1): %s", e)

173 if exceptions:

174 raise

175 try:

176 with open(full_path, "rb") as fp:

177 return fp.read()

178 except Exception as e:

179 logger.error("format_xml_file failed (2): %s", e)

180 return b""

181

182

183 def format_as_html_json(value: Any) -> str:

184 """

185 Returns value as JSON-formatted value in HTML.

186 :param value: Any value which can be converted to JSON by json.dumps

187 :return: str

188 """

189 return mark_safe(html.escape(json.dumps(value, indent=4)).replace("\n", "<br/>").replace(" ", "&nbsp;"))

190

191

192 def _format_dict_as_html_key(k: str) -> str:

193 if k.startswith("@"):

194 k = k[1:]

195 k = k.replace("_", " ")

196 k = re.sub(r"((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))", r" \1", k)

197 parts = k.split(" ")

198 out: List[str] = [str(capfirst(parts[0].strip()))]

199 for p in parts[1:]:

200 p2 = p.strip().lower()

201 if p2: 201 ↛ 199line 201 didn't jump to line 199, because the condition on line 201 was never false

202 out.append(p2)

203 return " ".join(out)

204

205

206 def _format_dict_as_html_r(data: Dict[str, Any], margin: str = "", format_keys: bool = True) -> str:

207 if not isinstance(data, dict): 207 ↛ 208line 207 didn't jump to line 208, because the condition on line 207 was never true

208 return "{}{}\n".format(margin, data)

209 out = ""

210 for k, v in OrderedDict(sorted(data.items())).items():

211 if isinstance(v, dict):

212 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k)

213 out += _format_dict_as_html_r(v, margin + " ", format_keys=format_keys)

214 out += "\n"

215 elif isinstance(v, list): 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true

216 for v2 in v:

217 out += "{}{}:\n".format(margin, _format_dict_as_html_key(k) if format_keys else k)

218 out += _format_dict_as_html_r(v2, margin + " ", format_keys=format_keys)

219 out += "\n"

220 else:

221 out += "{}{}: {}\n".format(margin, _format_dict_as_html_key(k) if format_keys else k, v)

222 return out

223

224

225 def format_dict_as_html(data: Dict[str, Any], format_keys: bool = True) -> str:

226 """

227 Formats dict to simple human readable pre-formatted html (<pre> tag).

228 :param data: dict

229 :param format_keys: Re-format 'additionalInfo' and 'additional_info' type of keys as 'Additional info'

230 :return: str (html)

231 """

232 return "<pre>" + _format_dict_as_html_r(data, format_keys=format_keys) + "</pre>"

233

234

235 def format_csv(rows: List[List[Any]], dialect: str = "excel") -> str:

236 """

237 Formats rows to CSV string content.

238 :param rows: List[List[Any]]

239 :param dialect: See csv.writer dialect

240 :return: str

241 """

242 f = StringIO()

243 writer = csv.writer(f, dialect=dialect)

244 for row in rows:

245 writer.writerow(row)

246 return f.getvalue()

247

248

249 def format_table( # noqa

250 rows: List[List[Any]],

251 max_col: Optional[int] = None,

252 max_line: Optional[int] = 200,

253 col_sep: str = "|",

254 row_sep: str = "-",

255 row_begin: str = "|",

256 row_end: str = "|",

257 has_label_row: bool = False,

258 left_align: Optional[List[int]] = None,

259 center_align: Optional[List[int]] = None,

260) -> str:

261 """

262 Formats "ASCII-table" rows by padding column widths to longest column value, optionally limiting column widths.

263 Optionally separates colums with ' | ' character and header row with '-' characters.

264 Supports left, right and center alignment. Useful for console apps / debugging.

265

266 :param rows: List[List[Any]]

267 :param max_col: Max column value width. Pass None for unlimited length.

268 :param max_line: Maximum single line length. Exceeding columns truncated. Pass None for unlimited length.

269 :param col_sep: Column separator string.

270 :param row_sep: Row separator character used before first row, end, after first row (if has_label_row).

271 :param row_begin: Row begin string, inserted before each row.

272 :param row_end: Row end string, appended after each row.

273 :param has_label_row: Set to True if table starts with column label row.

274 :param left_align: Indexes of left-aligned columns. By default all are right aligned.

275 :param center_align: Indexes of center-aligned columns. By default all are right aligned.

276 :return: str

277 """

278 # validate parameters

279 assert max_col is None or max_col > 2

280 if left_align is None:

281 left_align = []

282 if center_align is None:

283 center_align = []

284 if left_align:

285 if set(left_align) & set(center_align): 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true

286 raise ValidationError("Left align columns {} overlap with center align {}".format(left_align, center_align))

287

288 # find out number of columns

289 ncols = 0

290 for row in rows:

291 ncols = max(ncols, len(row))

292

293 # find out full-width column lengths

294 col_lens0: List[int] = [0] * ncols

295 for row in rows:

296 for ix, v in enumerate(row):

297 v = str(v)

298 col_lens0[ix] = max(col_lens0[ix], len(v))

299

300 # adjust max_col if needed

301 if max_line and (not max_col or sum(col_lens0) > max_line): 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true

302 max_col = max_line // ncols

303

304 # length limited lines and final column lengths

305 col_lens = [0] * ncols

306 lines: List[List[str]] = []

307 for row in rows:

308 line = []

309 for ix, v in enumerate(row):

310 v = str(v)

311 if max_col and len(v) > max_col:

312 v = v[: max_col - 2] + ".."

313 line.append(v)

314 col_lens[ix] = max(col_lens[ix], len(v))

315 while len(line) < ncols: 315 ↛ 316line 315 didn't jump to line 316, because the condition on line 315 was never true

316 line.append("")

317 lines.append(line)

318

319 # padded lines

320 lines2: List[List[str]] = []

321 for line in lines:

322 line2 = []

323 for ix, v in enumerate(line):

324 col_len = col_lens[ix]

325 if len(v) < col_len:

326 if ix in left_align:

327 v = v + " " * (col_len - len(v))

328 elif ix in center_align:

329 pad = col_len - len(v)

330 lpad = int(pad / 2)

331 rpad = pad - lpad

332 v = " " * lpad + v + " " * rpad

333 else:

334 v = " " * (col_len - len(v)) + v

335 line2.append(v)

336 lines2.append(line2)

337

338 # calculate max number of columns and max line length

339 max_line_len = 0

340 col_sep_len = len(col_sep)

341 ncols0 = ncols

342 for line in lines2:

343 if max_line is not None: 343 ↛ 342line 343 didn't jump to line 342, because the condition on line 343 was never false

344 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

345 while line_len > max_line:

346 ncols -= 1

347 line_len = len(row_begin) + sum(len(v) + col_sep_len for v in line[:ncols]) - col_sep_len + len(row_end)

348 max_line_len = max(max_line_len, line_len)

349

350 # find out how we should terminate lines/rows

351 line_term = ""

352 row_sep_term = ""

353 if ncols0 > ncols:

354 line_term = ".."

355 row_sep_term = row_sep * int(2 / len(row_sep))

356

357 # final output with row and column separators

358 lines3 = []

359 if row_sep: 359 ↛ 361line 359 didn't jump to line 361, because the condition on line 359 was never false

360 lines3.append(row_sep * max_line_len + row_sep_term)

361 for line_ix, line in enumerate(lines2):

362 while len(line) > ncols:

363 line.pop()

364 line_out = col_sep.join(line)

365 lines3.append(row_begin + line_out + row_end + line_term)

366 if line_ix == 0 and row_sep and has_label_row:

367 lines3.append(row_sep * max_line_len + row_sep_term)

368 if row_sep: 368 ↛ 370line 368 didn't jump to line 370, because the condition on line 368 was never false

369 lines3.append(row_sep * max_line_len + row_sep_term)

370 return "\n".join(lines3)

371

372

373 def ucfirst(v: str) -> str:

374 """

375 Converts first character of the string to uppercase.

376 :param v: str

377 :return: str

378 """

379 return v[0:1].upper() + v[1:]

380

381

382 ucfirst_lazy = lazy(ucfirst, str)

383

384

385 def dec1(a: Union[float, int, Decimal, str]) -> Decimal:

386 """

387 Converts number to Decimal with 1 decimal digits.

388 :param a: Number

389 :return: Decimal with 1 decimal digits

390 """

391 return Decimal(a).quantize(Decimal("1.0"))

392

393

394 def dec2(a: Union[float, int, Decimal, str]) -> Decimal:

395 """

396 Converts number to Decimal with 2 decimal digits.

397 :param a: Number

398 :return: Decimal with 2 decimal digits

399 """

400 return Decimal(a).quantize(Decimal("1.00"))

401

402

403 def dec3(a: Union[float, int, Decimal, str]) -> Decimal:

404 """

405 Converts number to Decimal with 3 decimal digits.

406 :param a: Number

407 :return: Decimal with 3 decimal digits

408 """

409 return Decimal(a).quantize(Decimal("1.000"))

410

411

412 def dec4(a: Union[float, int, Decimal, str]) -> Decimal:

413 """

414 Converts number to Decimal with 4 decimal digits.

415 :param a: Number

416 :return: Decimal with 4 decimal digits

417 """

418 return Decimal(a).quantize(Decimal("1.0000"))

419

420

421 def dec5(a: Union[float, int, Decimal, str]) -> Decimal:

422 """

423 Converts number to Decimal with 5 decimal digits.

424 :param a: Number

425 :return: Decimal with 4 decimal digits

426 """

427 return Decimal(a).quantize(Decimal("1.00000"))

428

429

430 def dec6(a: Union[float, int, Decimal, str]) -> Decimal:

431 """

432 Converts number to Decimal with 6 decimal digits.

433 :param a: Number

434 :return: Decimal with 4 decimal digits

435 """

436 return Decimal(a).quantize(Decimal("1.000000"))

437

438

439 def is_media_full_path(file_path: str) -> bool:

440 """

441 Checks if file path is under (settings) MEDIA_ROOT.

442 """

443 return (

444 hasattr(settings, "MEDIA_ROOT")

445 and settings.MEDIA_ROOT

446 and os.path.isabs(file_path)

447 and os.path.realpath(file_path).startswith(str(settings.MEDIA_ROOT))

448 )

449

450

451 def strip_media_root(file_path: str) -> str:

452 """

453 If file path starts with (settings) MEDIA_ROOT,

454 the MEDIA_ROOT part gets stripped and only relative path is returned.

455 Otherwise file path is returned as is. This enabled stored file names in more

456 portable format for different environment / storage.

457 If MEDIA_ROOT is missing or empty, the filename is returned as is.

458 Reverse operation of this is get_media_full_path().

459 :param file_path: str

460 :return: str

461 """

462 full_path = os.path.realpath(file_path)

463 if not is_media_full_path(full_path): 463 ↛ 464line 463 didn't jump to line 464, because the condition on line 463 was never true

464 logger.error("strip_media_root() expects absolute path under MEDIA_ROOT, got %s (%s)", file_path, full_path)

465 raise ValueError("strip_media_root() expects absolute path under MEDIA_ROOT")

466 file_path = full_path[len(settings.MEDIA_ROOT) :]

467 if file_path.startswith("/"): 467 ↛ 469line 467 didn't jump to line 469, because the condition on line 467 was never false

468 return file_path[1:]

469 return file_path

470

471

472 def get_media_full_path(file_path: str) -> str:

473 """

474 Returns the absolute path from a (relative) path to (settings) MEDIA_ROOT.

475 This enabled stored file names in more portable format for different environment / storage.

476 If MEDIA_ROOT is missing or non-media path is passed to function, exception is raised.

477 Reverse operation of this is strip_media_root().

478 :param file_path: str

479 :return: str

480 """

481 full_path = (

482 os.path.realpath(file_path) if os.path.isabs(file_path) else os.path.join(settings.MEDIA_ROOT, file_path)

483 )

484 if not is_media_full_path(full_path): 484 ↛ 485line 484 didn't jump to line 485, because the condition on line 484 was never true

485 logger.error("get_media_full_path() expects relative path to MEDIA_ROOT, got %s (%s)", file_path, full_path)

486 raise ValueError("get_media_full_path() expects relative path to MEDIA_ROOT")

487 return full_path

488

489

490 def camel_case_to_underscore(s: str) -> str:

491 """

492 Converts camelCaseWord to camel_case_word.

493 :param s: str

494 :return: str

495 """

496 if s: 496 ↛ 500line 496 didn't jump to line 500, because the condition on line 496 was never false

497 s = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", s)

498 s = re.sub(r"([a-z\d])([A-Z])", r"\1_\2", s)

499 s = s.replace("-", "_")

500 return s.lower()

501

502

503 def underscore_to_camel_case(s: str) -> str:

504 """

505 Converts under_score_word to underScoreWord.

506 :param s: str

507 :return: str

508 """

509 if s: 509 ↛ 512line 509 didn't jump to line 512, because the condition on line 509 was never false

510 p = s.split("_")

511 s = p[0] + "".join([ucfirst(w) for w in p[1:]])

512 return s

513

514

515 def choices_label(choices: Sequence[Tuple[S, str]], value: S) -> str:

516 """

517 Iterates (value,label) list and returns label matching the choice

518 :param choices: [(choice1, label1), (choice2, label2), ...]

519 :param value: Value to find

520 :return: label or None

521 """

522 for key, label in choices: 522 ↛ 525line 522 didn't jump to line 525, because the loop on line 522 didn't complete

523 if key == value: 523 ↛ 522line 523 didn't jump to line 522, because the condition on line 523 was never false

524 return label

525 return ""